From d085aca4037ba3da60d5d50c06d1a89c79730ec1 Mon Sep 17 00:00:00 2001 From: Vindhya Ningegowda Date: Wed, 30 Oct 2024 17:10:55 -0700 Subject: [PATCH 01/13] Rebase master --- api/src/main/java/io/grpc/NameResolver.java | 33 +- .../io/grpc/internal/ManagedChannelImpl.java | 5 +- .../InternalSharedXdsClientPoolProvider.java | 8 +- .../grpc/xds/SharedXdsClientPoolProvider.java | 24 +- .../io/grpc/xds/XdsClientMetricReporter.java | 99 ++ .../grpc/xds/XdsClientMetricReporterImpl.java | 187 +++ .../io/grpc/xds/XdsClientPoolFactory.java | 4 + .../java/io/grpc/xds/XdsNameResolver.java | 14 +- .../io/grpc/xds/XdsNameResolverProvider.java | 3 +- .../grpc/xds/client/ControlPlaneClient.java | 23 + .../java/io/grpc/xds/client/XdsClient.java | 12 + .../io/grpc/xds/client/XdsClientImpl.java | 123 +- .../java/io/grpc/xds/CsdsServiceTest.java | 6 + .../grpc/xds/GrpcXdsClientImplTestBase.java | 1066 +++++++++++++++-- .../xds/SharedXdsClientPoolProviderTest.java | 7 +- .../xds/XdsClientMetricReporterImplTest.java | 219 ++++ .../grpc/xds/XdsNameResolverProviderTest.java | 2 + .../java/io/grpc/xds/XdsNameResolverTest.java | 50 +- .../java/io/grpc/xds/XdsServerTestHelper.java | 7 + 19 files changed, 1784 insertions(+), 108 deletions(-) create mode 100644 xds/src/main/java/io/grpc/xds/XdsClientMetricReporter.java create mode 100644 xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java create mode 100644 xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java diff --git a/api/src/main/java/io/grpc/NameResolver.java b/api/src/main/java/io/grpc/NameResolver.java index b9590ab5d5a..58d49f559b0 100644 --- a/api/src/main/java/io/grpc/NameResolver.java +++ b/api/src/main/java/io/grpc/NameResolver.java @@ -288,6 +288,7 @@ public static final class Args { @Nullable private final ChannelLogger channelLogger; @Nullable private final Executor executor; @Nullable private final String overrideAuthority; + @Nullable private final MetricRecorder metricRecorder; private Args( Integer defaultPort, @@ -297,7 +298,8 @@ private Args( @Nullable ScheduledExecutorService scheduledExecutorService, @Nullable ChannelLogger channelLogger, @Nullable Executor executor, - @Nullable String overrideAuthority) { + @Nullable String overrideAuthority, + @Nullable MetricRecorder metricRecorder) { this.defaultPort = checkNotNull(defaultPort, "defaultPort not set"); this.proxyDetector = checkNotNull(proxyDetector, "proxyDetector not set"); this.syncContext = checkNotNull(syncContext, "syncContext not set"); @@ -306,6 +308,7 @@ private Args( this.channelLogger = channelLogger; this.executor = executor; this.overrideAuthority = overrideAuthority; + this.metricRecorder = metricRecorder; } /** @@ -403,6 +406,17 @@ public String getOverrideAuthority() { return overrideAuthority; } + /** + * Returns the {@link MetricRecorder} that the channel uses to record metrics. + * + * @since 1.67.0 + */ + @Nullable + @ExperimentalApi("Insert GitHub issue") + public MetricRecorder getMetricRecorder() { + return metricRecorder; + } + @Override public String toString() { @@ -415,6 +429,7 @@ public String toString() { .add("channelLogger", channelLogger) .add("executor", executor) .add("overrideAuthority", overrideAuthority) + .add("metricRecorder", metricRecorder) .toString(); } @@ -433,6 +448,7 @@ public Builder toBuilder() { builder.setChannelLogger(channelLogger); builder.setOffloadExecutor(executor); builder.setOverrideAuthority(overrideAuthority); + builder.setMetricRecorder(metricRecorder); return builder; } @@ -459,6 +475,7 @@ public static final class Builder { private ChannelLogger channelLogger; private Executor executor; private String overrideAuthority; + private MetricRecorder metricRecorder; Builder() { } @@ -545,6 +562,17 @@ public Builder setOverrideAuthority(String authority) { return this; } + /** + * See {@link Args#getMetricRecorder()}. This is an optional field. + * + * @since 1.67.0 + */ + @ExperimentalApi("Insert github issue") + public Builder setMetricRecorder(MetricRecorder metricRecorder) { + this.metricRecorder = metricRecorder; + return this; + } + /** * Builds an {@link Args}. * @@ -554,7 +582,8 @@ public Args build() { return new Args( defaultPort, proxyDetector, syncContext, serviceConfigParser, - scheduledExecutorService, channelLogger, executor, overrideAuthority); + scheduledExecutorService, channelLogger, executor, overrideAuthority, + metricRecorder); } } } diff --git a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java index b89a126517f..88fbf5e2c62 100644 --- a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java +++ b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java @@ -589,6 +589,8 @@ ClientStream newSubstream( builder.maxHedgedAttempts, loadBalancerFactory); this.authorityOverride = builder.authorityOverride; + this.metricRecorder = new MetricRecorderImpl(builder.metricSinks, + MetricInstrumentRegistry.getDefaultRegistry()); this.nameResolverArgs = NameResolver.Args.newBuilder() .setDefaultPort(builder.getDefaultPort()) @@ -599,6 +601,7 @@ ClientStream newSubstream( .setChannelLogger(channelLogger) .setOffloadExecutor(this.offloadExecutorHolder) .setOverrideAuthority(this.authorityOverride) + .setMetricRecorder(this.metricRecorder) .build(); this.nameResolver = getNameResolver( targetUri, authorityOverride, nameResolverProvider, nameResolverArgs); @@ -671,8 +674,6 @@ public CallTracer create() { } serviceConfigUpdated = true; } - this.metricRecorder = new MetricRecorderImpl(builder.metricSinks, - MetricInstrumentRegistry.getDefaultRegistry()); } @VisibleForTesting diff --git a/xds/src/main/java/io/grpc/xds/InternalSharedXdsClientPoolProvider.java b/xds/src/main/java/io/grpc/xds/InternalSharedXdsClientPoolProvider.java index 0073cce1a88..85b59fabfa0 100644 --- a/xds/src/main/java/io/grpc/xds/InternalSharedXdsClientPoolProvider.java +++ b/xds/src/main/java/io/grpc/xds/InternalSharedXdsClientPoolProvider.java @@ -17,6 +17,7 @@ package io.grpc.xds; import io.grpc.Internal; +import io.grpc.MetricRecorder; import io.grpc.internal.ObjectPool; import io.grpc.xds.client.XdsClient; import io.grpc.xds.client.XdsInitializationException; @@ -36,6 +37,11 @@ public static void setDefaultProviderBootstrapOverride(Map bootstrap) public static ObjectPool getOrCreate(String target) throws XdsInitializationException { - return SharedXdsClientPoolProvider.getDefaultProvider().getOrCreate(target); + return getOrCreate(target, new MetricRecorder() {}); + } + + public static ObjectPool getOrCreate(String target, MetricRecorder metricRecorder) + throws XdsInitializationException { + return SharedXdsClientPoolProvider.getDefaultProvider().getOrCreate(target, metricRecorder); } } diff --git a/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java b/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java index c9195896d82..88d015150e9 100644 --- a/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java +++ b/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java @@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; +import io.grpc.MetricRecorder; import io.grpc.internal.ExponentialBackoffPolicy; import io.grpc.internal.GrpcUtil; import io.grpc.internal.ObjectPool; @@ -56,6 +57,7 @@ final class SharedXdsClientPoolProvider implements XdsClientPoolFactory { private final Object lock = new Object(); private final AtomicReference> bootstrapOverride = new AtomicReference<>(); private final Map> targetToXdsClientMap = new ConcurrentHashMap<>(); + private final Map targetToMetricRecorderMap = new ConcurrentHashMap<>(); SharedXdsClientPoolProvider() { this(new GrpcBootstrapperImpl()); @@ -82,7 +84,14 @@ public ObjectPool get(String target) { } @Override + @Nullable public ObjectPool getOrCreate(String target) throws XdsInitializationException { + return this.getOrCreate(target, new MetricRecorder() {}); + } + + @Override + public ObjectPool getOrCreate(String target, MetricRecorder metricRecorder) + throws XdsInitializationException { ObjectPool ref = targetToXdsClientMap.get(target); if (ref == null) { synchronized (lock) { @@ -98,8 +107,12 @@ public ObjectPool getOrCreate(String target) throws XdsInitialization if (bootstrapInfo.servers().isEmpty()) { throw new XdsInitializationException("No xDS server provided"); } - ref = new RefCountedXdsClientObjectPool(bootstrapInfo, target); + MetricRecorder metricRecorderForTarget = targetToMetricRecorderMap.get(target); + metricRecorder = + metricRecorderForTarget != null ? metricRecorderForTarget : metricRecorder; + ref = new RefCountedXdsClientObjectPool(bootstrapInfo, target, metricRecorder); targetToXdsClientMap.put(target, ref); + targetToMetricRecorderMap.putIfAbsent(target, metricRecorder); } } } @@ -124,6 +137,7 @@ static class RefCountedXdsClientObjectPool implements ObjectPool { new ExponentialBackoffPolicy.Provider(); private final BootstrapInfo bootstrapInfo; private final String target; // The target associated with the xDS client. + private final MetricRecorder metricRecorder; private final Object lock = new Object(); @GuardedBy("lock") private ScheduledExecutorService scheduler; @@ -133,9 +147,11 @@ static class RefCountedXdsClientObjectPool implements ObjectPool { private int refCount; @VisibleForTesting - RefCountedXdsClientObjectPool(BootstrapInfo bootstrapInfo, String target) { + RefCountedXdsClientObjectPool(BootstrapInfo bootstrapInfo, String target, + MetricRecorder metricRecorder) { this.bootstrapInfo = checkNotNull(bootstrapInfo); this.target = target; + this.metricRecorder = metricRecorder; } @Override @@ -154,7 +170,9 @@ public XdsClient getObject() { GrpcUtil.STOPWATCH_SUPPLIER, TimeProvider.SYSTEM_TIME_PROVIDER, MessagePrinter.INSTANCE, - new TlsContextManagerImpl(bootstrapInfo)); + new TlsContextManagerImpl(bootstrapInfo), + getTarget(), + new XdsClientMetricReporterImpl(metricRecorder)); } refCount++; return xdsClient; diff --git a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporter.java b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporter.java new file mode 100644 index 00000000000..09113f12f06 --- /dev/null +++ b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporter.java @@ -0,0 +1,99 @@ +/* + * Copyright 2024 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +import io.grpc.Internal; +import io.grpc.xds.client.XdsClient; + +/** + * Interface for reporting metrics from the xDS client. + * We need this indirection, to de couple Xds from OpenTelemetry + */ +@Internal +public interface XdsClientMetricReporter { + + /** + * Reports resource update counts. + * + * @param validResourceCount the number of resources that were successfully updated. + * @param invalidResourceCount the number of resources that failed to update. + * @param target the xDS management server name for the load balancing policy this update is for. + * @param xdsServer the xDS management server address for this update. + * @param resourceType the type of resource (e.g., "LDS", "RDS", "CDS", "EDS"). + */ + default void reportResourceUpdates(long validResourceCount, long invalidResourceCount, + String target, String xdsServer, String resourceType) { + } + + /** + * Reports xDS server failure counts. + * + * @param serverFailure the number of times the xDS server has failed. + * @param target the xDS management server name for the load balancing policy this failure is for. + * @param xdsServer the xDS management server address for this failure. + */ + default void reportServerFailure(long serverFailure, String target, String xdsServer) { + } + + /** + * Sets the {@link XdsClient} instance to the reporter. + * + * @param xdsClient the {@link XdsClient} instance. + */ + default void setXdsClient(XdsClient xdsClient) { + } + + /** + * Closes the metric reporter. + */ + default void close() { + } + + /** + * Interface for reporting metrics from the xDS client callbacks. + * + */ + interface CallbackMetricReporter { + + /** + * Reports resource counts in the cache. + * + * @param resourceCount the number of resources in the cache. + * @param cacheState the state of the cache (e.g., "SYNCED", "DOES_NOT_EXIST"). + * @param resourceType the type of resource (e.g., "LDS", "RDS", "CDS", "EDS"). + * @param target the xDS management server name for the load balancing policy this count is + * for. + */ + // TODO(@dnvindhya): include the "authority" label once authority is available. + default void reportResourceCounts(long resourceCount, String cacheState, String resourceType, + String target) { + } + + /** + * Reports server connection status. + * + * @param isConnected {@code true} if the client is connected to the xDS server, {@code false} + * otherwise. + * @param target the xDS management server name for the load balancing policy this connection + * is for. + * @param xdsServer the xDS management server address for this connection. + * @since 0.1.0 + */ + default void reportServerConnections(int isConnected, String target, String xdsServer) { + } + } +} diff --git a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java new file mode 100644 index 00000000000..1188e3da1d6 --- /dev/null +++ b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java @@ -0,0 +1,187 @@ +/* + * Copyright 2024 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.SettableFuture; +import io.grpc.Internal; +import io.grpc.LongCounterMetricInstrument; +import io.grpc.LongGaugeMetricInstrument; +import io.grpc.MetricInstrumentRegistry; +import io.grpc.MetricRecorder; +import io.grpc.MetricRecorder.BatchCallback; +import io.grpc.MetricRecorder.BatchRecorder; +import io.grpc.MetricRecorder.Registration; +import io.grpc.xds.client.XdsClient; +import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.annotation.Nullable; + +/** + * XdsClientMetricReporter implementation. + */ +@Internal +public class XdsClientMetricReporterImpl implements XdsClientMetricReporter { + + private static final Logger logger = Logger.getLogger( + XdsClientMetricReporterImpl.class.getName()); + private static final LongCounterMetricInstrument SERVER_FAILURE_COUNTER; + private static final LongCounterMetricInstrument RESOURCE_UPDATES_VALID_COUNTER; + private static final LongCounterMetricInstrument RESOURCE_UPDATES_INVALID_COUNTER; + private static final LongGaugeMetricInstrument CONNECTED_GAUGE; + private static final LongGaugeMetricInstrument RESOURCES_GAUGE; + + private final MetricRecorder metricRecorder; + @Nullable + private Registration gaugeRegistration = null; + @Nullable + private XdsClient xdsClient = null; + + static { + MetricInstrumentRegistry metricInstrumentRegistry + = MetricInstrumentRegistry.getDefaultRegistry(); + SERVER_FAILURE_COUNTER = metricInstrumentRegistry.registerLongCounter( + "grpc.xds_client.server_failure", + "EXPERIMENTAL. A counter of xDS servers going from healthy to unhealthy. A server goes" + + " unhealthy when we have a connectivity failure or when the ADS stream fails without" + + " seeing a response message, as per gRFC A57.", "{failure}", + Arrays.asList("grpc.target", "grpc.xds.server"), Collections.emptyList(), false); + RESOURCE_UPDATES_VALID_COUNTER = metricInstrumentRegistry.registerLongCounter( + "grpc.xds_client.resource_updates_valid", + "EXPERIMENTAL. A counter of resources received that were considered valid. The counter will" + + " be incremented even for resources that have not changed.", "{resource}", + Arrays.asList("grpc.target", "grpc.xds.server", "grpc.xds.resource_type"), + Collections.emptyList(), false); + RESOURCE_UPDATES_INVALID_COUNTER = metricInstrumentRegistry.registerLongCounter( + "grpc.xds_client.resource_updates_invalid", + "EXPERIMENTAL. A counter of resources received that were considered invalid.", "{resource}", + Arrays.asList("grpc.target", "grpc.xds.server", "grpc.xds.resource_type"), + Collections.emptyList(), false); + CONNECTED_GAUGE = metricInstrumentRegistry.registerLongGauge("grpc.xds_client.connected", + "EXPERIMENTAL. Whether or not the xDS client currently has a working ADS stream to the xDS" + + " server. For a given server, this will be set to 1 when the stream is initially" + + " created. It will be set to 0 when we have a connectivity failure or when the ADS" + + " stream fails without seeing a response message, as per gRFC A57. Once set to 0, it" + + " will be reset to 1 when we receive the first response on an ADS stream.", "{bool}", + Arrays.asList("grpc.target", "grpc.xds.server"), Collections.emptyList(), false); + RESOURCES_GAUGE = metricInstrumentRegistry.registerLongGauge("grpc.xds_client.resources", + "EXPERIMENTAL. Number of xDS resources.", "{resource}", + Arrays.asList("grpc.target", "grpc.xds.authority", "grpc.xds.cache_state", + "grpc.xds.resource_type"), Collections.emptyList(), false); + } + + XdsClientMetricReporterImpl(MetricRecorder metricRecorder) { + this.metricRecorder = metricRecorder; + } + + @Override + public void reportResourceUpdates(long validResourceCount, long invalidResourceCount, + String target, String xdsServer, String resourceType) { + metricRecorder.addLongCounter(RESOURCE_UPDATES_VALID_COUNTER, validResourceCount, + Arrays.asList(target, xdsServer, resourceType), Collections.emptyList()); + metricRecorder.addLongCounter(RESOURCE_UPDATES_INVALID_COUNTER, invalidResourceCount, + Arrays.asList(target, xdsServer, resourceType), Collections.emptyList()); + } + + @Override + public void reportServerFailure(long serverFailure, String target, String xdsServer) { + metricRecorder.addLongCounter(SERVER_FAILURE_COUNTER, serverFailure, + Arrays.asList(target, xdsServer), Collections.emptyList()); + } + + @Override + public void close() { + if (gaugeRegistration != null) { + gaugeRegistration.close(); + } + } + + @Override + public void setXdsClient(XdsClient client) { + this.xdsClient = client; + // register gauge here + this.gaugeRegistration = metricRecorder.registerBatchCallback(new BatchCallback() { + @Override + public void accept(BatchRecorder recorder) { + reportCallbackMetrics(recorder); + } + }, CONNECTED_GAUGE, RESOURCES_GAUGE); + } + + // ... (other methods) + + @VisibleForTesting + void reportCallbackMetrics(BatchRecorder recorder) { + CallbackMetricReporter callbackMetricReporter = new CallbackMetricReporterImpl(recorder); + try { + reportResourceCounts(callbackMetricReporter); + reportServerConnections(callbackMetricReporter); + } catch (Exception e) { + logger.log(Level.WARNING, "Reporting gauge metrics failed", e); + } + } + + /** + * Reports resource counts. + * This method is extracted for better testability. + */ + @VisibleForTesting + void reportResourceCounts(CallbackMetricReporter callbackMetricReporter) throws Exception { + SettableFuture ret = this.xdsClient.reportResourceCounts( + callbackMetricReporter); + // Normally this shouldn't take long, but adding a timeout to avoid indefinite blocking + ret.get(5, TimeUnit.SECONDS); + } + + /** + * Reports server connections. + * This method is extracted for better testability. + */ + @VisibleForTesting + void reportServerConnections(CallbackMetricReporter callbackMetricReporter) throws Exception { + SettableFuture ret = this.xdsClient.reportServerConnections(callbackMetricReporter); + // Normally this shouldn't take long, but adding a timeout to avoid indefinite blocking + ret.get(5, TimeUnit.SECONDS); + } + + @VisibleForTesting + static final class CallbackMetricReporterImpl implements + XdsClientMetricReporter.CallbackMetricReporter { + private final BatchRecorder recorder; + + CallbackMetricReporterImpl(BatchRecorder recorder) { + this.recorder = recorder; + } + + @Override + public void reportServerConnections(int isConnected, String target, String xdsServer) { + recorder.recordLongGauge(CONNECTED_GAUGE, isConnected, Arrays.asList(target, xdsServer), + Collections.emptyList()); + } + + // TODO(@dnvindhya): include the "authority" label once authority is available. + @Override + public void reportResourceCounts(long resourceCount, String cacheState, String resourceType, + String target) { + recorder.recordLongGauge(RESOURCES_GAUGE, resourceCount, + Arrays.asList(target, cacheState, resourceType), Collections.emptyList()); + } + } +} diff --git a/xds/src/main/java/io/grpc/xds/XdsClientPoolFactory.java b/xds/src/main/java/io/grpc/xds/XdsClientPoolFactory.java index 313eb675116..ec2dc373d57 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientPoolFactory.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientPoolFactory.java @@ -16,6 +16,7 @@ package io.grpc.xds; +import io.grpc.MetricRecorder; import io.grpc.internal.ObjectPool; import io.grpc.xds.client.XdsClient; import io.grpc.xds.client.XdsInitializationException; @@ -31,5 +32,8 @@ interface XdsClientPoolFactory { ObjectPool getOrCreate(String target) throws XdsInitializationException; + ObjectPool getOrCreate(String target, MetricRecorder metricRecorder) + throws XdsInitializationException; + List getTargets(); } diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java index 0beb6dc2483..7f576d48711 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java @@ -41,6 +41,7 @@ import io.grpc.LoadBalancer.PickSubchannelArgs; import io.grpc.Metadata; import io.grpc.MethodDescriptor; +import io.grpc.MetricRecorder; import io.grpc.NameResolver; import io.grpc.Status; import io.grpc.Status.Code; @@ -127,6 +128,7 @@ final class XdsNameResolver extends NameResolver { private final long randomChannelId; private volatile RoutingConfig routingConfig = RoutingConfig.empty; + private volatile MetricRecorder metricRecorder; private Listener2 listener; private ObjectPool xdsClientPool; private XdsClient xdsClient; @@ -140,10 +142,12 @@ final class XdsNameResolver extends NameResolver { URI targetUri, String name, @Nullable String overrideAuthority, ServiceConfigParser serviceConfigParser, SynchronizationContext syncContext, ScheduledExecutorService scheduler, - @Nullable Map bootstrapOverride) { + @Nullable Map bootstrapOverride, + @Nullable MetricRecorder metricRecorder) { this(targetUri, targetUri.getAuthority(), name, overrideAuthority, serviceConfigParser, syncContext, scheduler, SharedXdsClientPoolProvider.getDefaultProvider(), - ThreadSafeRandomImpl.instance, FilterRegistry.getDefaultRegistry(), bootstrapOverride); + ThreadSafeRandomImpl.instance, FilterRegistry.getDefaultRegistry(), bootstrapOverride, + metricRecorder); } @VisibleForTesting @@ -152,7 +156,8 @@ final class XdsNameResolver extends NameResolver { @Nullable String overrideAuthority, ServiceConfigParser serviceConfigParser, SynchronizationContext syncContext, ScheduledExecutorService scheduler, XdsClientPoolFactory xdsClientPoolFactory, ThreadSafeRandom random, - FilterRegistry filterRegistry, @Nullable Map bootstrapOverride) { + FilterRegistry filterRegistry, @Nullable Map bootstrapOverride, + @Nullable MetricRecorder metricRecorder) { this.targetAuthority = targetAuthority; target = targetUri.toString(); @@ -170,6 +175,7 @@ final class XdsNameResolver extends NameResolver { this.xdsClientPoolFactory.setBootstrapOverride(bootstrapOverride); this.random = checkNotNull(random, "random"); this.filterRegistry = checkNotNull(filterRegistry, "filterRegistry"); + this.metricRecorder = metricRecorder; randomChannelId = random.nextLong(); logId = InternalLogId.allocate("xds-resolver", name); logger = XdsLogger.withLogId(logId); @@ -185,7 +191,7 @@ public String getServiceAuthority() { public void start(Listener2 listener) { this.listener = checkNotNull(listener, "listener"); try { - xdsClientPool = xdsClientPoolFactory.getOrCreate(target); + xdsClientPool = xdsClientPoolFactory.getOrCreate(target, metricRecorder); } catch (Exception e) { listener.onError( Status.UNAVAILABLE.withDescription("Failed to initialize xDS").withCause(e)); diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider.java b/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider.java index 8d0e59eaa91..74518331269 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider.java @@ -81,7 +81,8 @@ public XdsNameResolver newNameResolver(URI targetUri, Args args) { targetUri, name, args.getOverrideAuthority(), args.getServiceConfigParser(), args.getSynchronizationContext(), args.getScheduledExecutorService(), - bootstrapOverride); + bootstrapOverride, + args.getMetricRecorder()); } return null; } diff --git a/xds/src/main/java/io/grpc/xds/client/ControlPlaneClient.java b/xds/src/main/java/io/grpc/xds/client/ControlPlaneClient.java index 5ac979277c4..f33f54e441d 100644 --- a/xds/src/main/java/io/grpc/xds/client/ControlPlaneClient.java +++ b/xds/src/main/java/io/grpc/xds/client/ControlPlaneClient.java @@ -78,6 +78,7 @@ final class ControlPlaneClient { private final Map, String> versions = new HashMap<>(); private boolean shutdown; + private boolean streamClosedNoResponse; @Nullable private AdsStream adsStream; @Nullable @@ -224,6 +225,25 @@ void readyHandler() { xdsClient.startSubscriberTimersIfNeeded(serverInfo); } + /** + * Indicates whether there is an active ADS stream. + * + *

Return {@code true} when the {@code AdsStream} is created. + * {@code false} when the ADS stream fails without a response. Resets to true + * upon receiving the first response on a new ADS stream. + */ + // Must be synchronized + boolean hasWorkingAdsStream() { + if (streamClosedNoResponse || shutdown) { + return false; + } + if (adsStream == null) { + return true; + } + return adsStream.responseReceived || !adsStream.closed; + } + + /** * Establishes the RPC connection by creating a new RPC stream on the given channel for * xDS protocol communication. @@ -332,6 +352,8 @@ public void onRecvMessage(DiscoveryResponse response) { syncContext.execute(new Runnable() { @Override public void run() { + // Reset flag as message has been received on a stream + streamClosedNoResponse = false; XdsResourceType type = fromTypeUrl(response.getTypeUrl()); if (logger.isLoggable(XdsLogLevel.DEBUG)) { logger.log( @@ -408,6 +430,7 @@ private void handleRpcStreamClosed(Status status) { "ADS stream closed by server after a response was received"); } } else { + streamClosedNoResponse = true; // If the ADS stream is closed without ever having received a response from the server, then // the XdsClient should consider that a connectivity error (see gRFC A57). if (status.isOk()) { diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClient.java b/xds/src/main/java/io/grpc/xds/client/XdsClient.java index fc7e1777384..0e9c2341f50 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClient.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClient.java @@ -24,9 +24,11 @@ import com.google.common.net.UrlEscapers; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.SettableFuture; import com.google.protobuf.Any; import io.grpc.ExperimentalApi; import io.grpc.Status; +import io.grpc.xds.XdsClientMetricReporter.CallbackMetricReporter; import io.grpc.xds.client.Bootstrapper.ServerInfo; import java.net.URI; import java.net.URISyntaxException; @@ -378,6 +380,16 @@ public Map getServerLrsClientMap() { throw new UnsupportedOperationException(); } + public SettableFuture reportResourceCounts( + CallbackMetricReporter callbackMetricReporter) { + throw new UnsupportedOperationException(); + } + + public SettableFuture reportServerConnections( + CallbackMetricReporter callbackMetricReporter) { + throw new UnsupportedOperationException(); + } + static final class ProcessingTracker { private final AtomicInteger pendingTask = new AtomicInteger(1); private final Executor executor; diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java index e2380b9ed73..bd324a40f71 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java @@ -38,16 +38,19 @@ import io.grpc.SynchronizationContext.ScheduledHandle; import io.grpc.internal.BackoffPolicy; import io.grpc.internal.TimeProvider; +import io.grpc.xds.XdsClientMetricReporter; +import io.grpc.xds.XdsClientMetricReporter.CallbackMetricReporter; import io.grpc.xds.client.Bootstrapper.AuthorityInfo; import io.grpc.xds.client.Bootstrapper.ServerInfo; +import io.grpc.xds.client.XdsClient.ResourceMetadata.ResourceMetadataStatus; import io.grpc.xds.client.XdsClient.ResourceStore; -import io.grpc.xds.client.XdsClient.XdsResponseHandler; import io.grpc.xds.client.XdsLogger.XdsLogLevel; import java.net.URI; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -60,7 +63,7 @@ * XdsClient implementation. */ @Internal -public final class XdsClientImpl extends XdsClient implements XdsResponseHandler, ResourceStore { +public final class XdsClientImpl extends XdsClient implements ResourceStore { // Longest time to wait, since the subscription to some resource, for concluding its absence. @VisibleForTesting @@ -100,6 +103,8 @@ public void uncaughtException(Thread t, Throwable e) { private final XdsLogger logger; private volatile boolean isShutdown; private final MessagePrettyPrinter messagePrinter; + private final String target; + private final XdsClientMetricReporter metricReporter; public XdsClientImpl( XdsTransportFactory xdsTransportFactory, @@ -109,7 +114,9 @@ public XdsClientImpl( Supplier stopwatchSupplier, TimeProvider timeProvider, MessagePrettyPrinter messagePrinter, - Object securityConfig) { + Object securityConfig, + String target, + XdsClientMetricReporter metricReporter) { this.xdsTransportFactory = xdsTransportFactory; this.bootstrapInfo = bootstrapInfo; this.timeService = timeService; @@ -118,13 +125,15 @@ public XdsClientImpl( this.timeProvider = timeProvider; this.messagePrinter = messagePrinter; this.securityConfig = securityConfig; + this.target = target; + this.metricReporter = metricReporter; + metricReporter.setXdsClient(this); logId = InternalLogId.allocate("xds-client", null); logger = XdsLogger.withLogId(logId); logger.log(XdsLogLevel.INFO, "Created"); } - @Override - public void handleResourceResponse( + private void handleResourceResponse( XdsResourceType xdsResourceType, ServerInfo serverInfo, String versionInfo, List resources, String nonce, ProcessingTracker processingTracker) { checkNotNull(xdsResourceType, "xdsResourceType"); @@ -138,11 +147,11 @@ public void handleResourceResponse( handleResourceUpdate(args, resources, xdsResourceType, processingTracker); } - @Override - public void handleStreamClosed(Status error) { + private void handleStreamClosed(Status error, ServerInfo serverInfo) { syncContext.throwIfNotInThisSynchronizationContext(); cleanUpResourceTimers(); if (!error.isOk()) { + metricReporter.reportServerFailure(1L, target, serverInfo.target()); for (Map> subscriberMap : resourceSubscribers.values()) { for (ResourceSubscriber subscriber : subscriberMap.values()) { @@ -154,8 +163,7 @@ public void handleStreamClosed(Status error) { } } - @Override - public void handleStreamRestarted(ServerInfo serverInfo) { + private void handleStreamRestarted(ServerInfo serverInfo) { syncContext.throwIfNotInThisSynchronizationContext(); for (Map> subscriberMap : resourceSubscribers.values()) { @@ -183,6 +191,7 @@ public void run() { for (final LoadReportClient lrsClient : serverLrsClientMap.values()) { lrsClient.stopLoadReporting(); } + metricReporter.close(); cleanUpResourceTimers(); } }); @@ -394,7 +403,27 @@ public ControlPlaneClient getOrCreateControlPlaneClient(ServerInfo serverInfo) { xdsTransport, serverInfo, bootstrapInfo.node(), - this, + new XdsResponseHandler() { + + @Override + public void handleResourceResponse( + XdsResourceType resourceType, ServerInfo serverInfo, String versionInfo, + List resources, String nonce, ProcessingTracker processingTracker) { + XdsClientImpl.this.handleResourceResponse(resourceType, serverInfo, versionInfo, + resources, nonce, + processingTracker); + } + + @Override + public void handleStreamClosed(Status error) { + XdsClientImpl.this.handleStreamClosed(error, serverInfo); + } + + @Override + public void handleStreamRestarted(ServerInfo serverInfo) { + XdsClientImpl.this.handleStreamRestarted(serverInfo); + } + }, this, timeService, syncContext, @@ -448,6 +477,10 @@ private void handleResourceUpdate( xdsResourceType.typeName(), args.versionInfo, args.nonce, result.unpackedResources); Map> parsedResources = result.parsedResources; Set invalidResources = result.invalidResources; + metricReporter.reportResourceUpdates(Long.valueOf(parsedResources.size()), + Long.valueOf(invalidResources.size()), target, + args.getServerInfo().target(), xdsResourceType.typeUrl()); + List errors = result.errors; String errorDetail = null; if (errors.isEmpty()) { @@ -504,6 +537,76 @@ private void handleResourceUpdate( } } + @Override + public SettableFuture reportServerConnections( + CallbackMetricReporter callbackMetricReporter) { + SettableFuture future = SettableFuture.create(); + syncContext.execute(() -> { + serverCpClientMap.forEach((serverInfo, controlPlaneClient) -> + callbackMetricReporter.reportServerConnections( + controlPlaneClient.hasWorkingAdsStream() ? 1 : 0, + target, + serverInfo.target())); + future.set(null); + }); + return future; + } + + @Override + public SettableFuture reportResourceCounts(CallbackMetricReporter callbackMetricReporter) { + SettableFuture future = SettableFuture.create(); + syncContext.execute(() -> { + Map, Map> resourceCountsByType = + getResourceCountsByType(); + reportResourceCountsToCallback(callbackMetricReporter, resourceCountsByType); + future.set(null); + }); + return future; + } + + private String cacheStateFromResourceStatus(ResourceMetadata metadata, boolean isResourceCached) { + String status = metadata.getStatus().toString().toLowerCase(Locale.ROOT); + return metadata.getStatus() == ResourceMetadataStatus.NACKED && isResourceCached + ? status + "_but_cached" : status; + } + + /** + * Calculates resource counts by ResourceType and ResourceSubscriber.metadata.status + */ + Map, Map> getResourceCountsByType() { + Map, Map> resourceCountsByType = new HashMap<>(); + for (XdsResourceType resourceType : resourceSubscribers.keySet()) { + Map resourceCountsByState = new HashMap<>(); + for (ResourceSubscriber subscriber : + resourceSubscribers.get(resourceType).values()) { + String cacheState = cacheStateFromResourceStatus(subscriber.metadata, + subscriber.data != null); + resourceCountsByState.compute(cacheState, (k, v) -> (v == null) ? 1 : v + 1); + } + resourceCountsByType.put(resourceType, resourceCountsByState); + } + return resourceCountsByType; + } + + /** + * Reports resource counts using the provided callbackMetricReporter. + */ + void reportResourceCountsToCallback(CallbackMetricReporter callbackMetricReporter, + Map, Map> resourceCountsByType) { + for (Map.Entry, Map> entry : + resourceCountsByType.entrySet()) { + XdsResourceType resourceType = entry.getKey(); + Map resourceCountsByState = entry.getValue(); + // TODO(@dnvindhya): include the "authority" label once authority is available here. + resourceCountsByState.forEach((cacheState, count) -> + callbackMetricReporter.reportResourceCounts( + count, + cacheState, resourceType.typeUrl(), target + )); + } + } + + /** * Tracks a single subscribed resource. */ diff --git a/xds/src/test/java/io/grpc/xds/CsdsServiceTest.java b/xds/src/test/java/io/grpc/xds/CsdsServiceTest.java index 8166027033f..8002944eff2 100644 --- a/xds/src/test/java/io/grpc/xds/CsdsServiceTest.java +++ b/xds/src/test/java/io/grpc/xds/CsdsServiceTest.java @@ -39,6 +39,7 @@ import io.envoyproxy.envoy.type.matcher.v3.NodeMatcher; import io.grpc.Deadline; import io.grpc.InsecureChannelCredentials; +import io.grpc.MetricRecorder; import io.grpc.Status; import io.grpc.Status.Code; import io.grpc.StatusRuntimeException; @@ -557,5 +558,10 @@ public void setBootstrapOverride(Map bootstrap) { public ObjectPool getOrCreate(String target) { throw new UnsupportedOperationException("Should not be called"); } + + @Override + public ObjectPool getOrCreate(String target, MetricRecorder metricRecorder) { + throw new UnsupportedOperationException("Should not be called"); + } } } diff --git a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java index 8ecb40383b1..0b41b2b1e32 100644 --- a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java +++ b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java @@ -112,6 +112,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.Executor; +import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -142,6 +143,7 @@ // The base class was used to test both xds v2 and v3. V2 is dropped now so the base class is not // necessary. Still keep it for future version usage. Remove if too much trouble to maintain. public abstract class GrpcXdsClientImplTestBase { + private static final String SERVER_URI = "trafficdirector.googleapis.com"; private static final String SERVER_URI_CUSTOME_AUTHORITY = "trafficdirector2.googleapis.com"; private static final String SERVER_URI_EMPTY_AUTHORITY = "trafficdirector3.googleapis.com"; @@ -155,6 +157,7 @@ public abstract class GrpcXdsClientImplTestBase { private static final String VERSION_2 = "43"; private static final String VERSION_3 = "44"; private static final String NODE_ID = "cool-node-id"; + private static final String CHANNEL_TARGET = "target"; private static final Node NODE = Node.newBuilder().setId(NODE_ID).build(); private static final Any FAILING_ANY = MessageFactory.FAILING_ANY; private static final ChannelCredentials CHANNEL_CREDENTIALS = InsecureChannelCredentials.create(); @@ -287,6 +290,10 @@ public long currentTimeNanos() { private ResourceWatcher cdsResourceWatcher; @Mock private ResourceWatcher edsResourceWatcher; + @Mock + private XdsClientMetricReporter xdsClientMetricReporter; + @Mock + private XdsClientMetricReporter.CallbackMetricReporter callbackMetricReporter; private ManagedChannel channel; private ManagedChannel channelForCustomAuthority; @@ -369,7 +376,9 @@ public XdsTransport create(ServerInfo serverInfo) { fakeClock.getStopwatchSupplier(), timeProvider, MessagePrinter.INSTANCE, - new TlsContextManagerImpl(bootstrapInfo)); + new TlsContextManagerImpl(bootstrapInfo), + CHANNEL_TARGET, + xdsClientMetricReporter); assertThat(resourceDiscoveryCalls).isEmpty(); assertThat(loadReportCalls).isEmpty(); @@ -602,6 +611,44 @@ private void validateGoldenClusterLoadAssignment(EdsUpdate edsUpdate) { LocalityLbEndpoints.create(ImmutableList.of(), 2, 1)); } + /** + * Verifies that the {@link XdsClientMetricReporter#reportResourceUpdates} method has been called + * the expected number of times with the expected values for valid resource count, invalid + * resource count, and corresponding metric labels. + */ + private void verifyResourceValidInvalidMetrics(int times, long validResourceCount, + long invalidResourceCount, String dataPlaneTargetLabel, String xdsServerTargetLabel, + String resourceType) { + verify(xdsClientMetricReporter, times(times)).reportResourceUpdates( + eq(validResourceCount), + eq(invalidResourceCount), + eq(dataPlaneTargetLabel), + eq(xdsServerTargetLabel), + eq(resourceType)); + } + + private void reportResourceCount() { + try { + Future unused = xdsClient.reportResourceCounts(callbackMetricReporter); + } catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new AssertionError(e); + } + } + + private void reportServerConnection() { + try { + Future unused = xdsClient.reportServerConnections(callbackMetricReporter); + } catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new AssertionError(e); + } + } + @Test public void ldsResourceNotFound() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), LDS_RESOURCE, @@ -615,13 +662,20 @@ public void ldsResourceNotFound() { call.verifyRequest(LDS, LDS_RESOURCE, VERSION_1, "0000", NODE); verifyNoInteractions(ldsResourceWatcher); verifyResourceMetadataRequested(LDS, LDS_RESOURCE); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), + CHANNEL_TARGET); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); // Server failed to return subscribed resource within expected time window. fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); verify(ldsResourceWatcher).onResourceDoesNotExist(LDS_RESOURCE); assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); verifyResourceMetadataDoesNotExist(LDS, LDS_RESOURCE); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "does_not_exist", LDS.typeUrl(), + CHANNEL_TARGET); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -682,8 +736,9 @@ public void ldsResponseErrorHandling_someResourcesFailedUnpack() { /** * Tests a subscribed LDS resource transitioned to and from the invalid state. * - * @see - * A40-csds-support.md + * @see + * A40-csds-support.md */ @Test public void ldsResponseErrorHandling_subscribedResourceInvalid() { @@ -696,6 +751,10 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataRequested(LDS, "A"); verifyResourceMetadataRequested(LDS, "B"); verifyResourceMetadataRequested(LDS, "C"); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(3, "requested", LDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(3, 0, 0, 0); // LDS -> {A, B, C}, version 1 @@ -705,9 +764,14 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid() { "C", Any.pack(mf.buildListenerWithApiListenerForRds("C", "C.1"))); call.sendResponse(LDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 + verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + LDS.typeUrl()); verifyResourceMetadataAcked(LDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(3, "acked", LDS.typeUrl(), CHANNEL_TARGET + ); call.verifyRequest(LDS, subscribedResourceNames, VERSION_1, "0000", NODE); // LDS -> {A, B}, version 2 @@ -719,15 +783,32 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid() { // {A} -> ACK, version 2 // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> does not exist + verifyResourceValidInvalidMetrics(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), + LDS.typeUrl()); List errorsV2 = ImmutableList.of("LDS response Listener 'B' validation error: "); verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataNacked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, VERSION_2, TIME_INCREMENT * 2, errorsV2); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(LDS, "C"); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", LDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(1, "does_not_exist", LDS.typeUrl(), + CHANNEL_TARGET + ); } else { // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(2, "acked", LDS.typeUrl(), CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", LDS.typeUrl(), + CHANNEL_TARGET + ); } call.verifyRequestNack(LDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); @@ -738,16 +819,29 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid() { call.sendResponse(LDS, resourcesV3.values().asList(), VERSION_3, "0002"); // {A} -> does not exist // {B, C} -> ACK, version 3 + verifyResourceValidInvalidMetrics(1, 2, 0, CHANNEL_TARGET, xdsServerInfo.target(), + LDS.typeUrl()); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(LDS, "A"); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "does_not_exist", + LDS.typeUrl(), CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(2, "acked", LDS.typeUrl(), CHANNEL_TARGET + ); } else { // When resource deletion is disabled, {A} stays ACKed in the previous version VERSION_2. verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", LDS.typeUrl(), + CHANNEL_TARGET + ); } verifyResourceMetadataAcked(LDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); verifyResourceMetadataAcked(LDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); call.verifyRequest(LDS, subscribedResourceNames, VERSION_3, "0002", NODE); verifySubscribedResourcesMetadataSizes(3, 0, 0, 0); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -767,6 +861,13 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti verifyResourceMetadataRequested(RDS, "A.1"); verifyResourceMetadataRequested(RDS, "B.1"); verifyResourceMetadataRequested(RDS, "C.1"); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(3, "requested", LDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(3, "requested", RDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(3, 0, 3, 0); // LDS -> {A, B, C}, version 1 @@ -776,9 +877,16 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti "C", Any.pack(mf.buildListenerWithApiListenerForRds("C", "C.1"))); call.sendResponse(LDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 + verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + LDS.typeUrl()); verifyResourceMetadataAcked(LDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(3, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "requested", RDS.typeUrl(), + CHANNEL_TARGET + ); call.verifyRequest(LDS, subscribedResourceNames, VERSION_1, "0000", NODE); // RDS -> {A.1, B.1, C.1}, version 1 @@ -789,9 +897,15 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti "C.1", Any.pack(mf.buildRouteConfiguration("C.1", vhostsV1))); call.sendResponse(RDS, resourcesV11.values().asList(), VERSION_1, "0000"); // {A.1, B.1, C.1} -> ACK, version 1 + verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + RDS.typeUrl()); verifyResourceMetadataAcked(RDS, "A.1", resourcesV11.get("A.1"), VERSION_1, TIME_INCREMENT * 2); verifyResourceMetadataAcked(RDS, "B.1", resourcesV11.get("B.1"), VERSION_1, TIME_INCREMENT * 2); verifyResourceMetadataAcked(RDS, "C.1", resourcesV11.get("C.1"), VERSION_1, TIME_INCREMENT * 2); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", LDS.typeUrl(), + CHANNEL_TARGET); + verify(callbackMetricReporter).reportResourceCounts(3, "acked", RDS.typeUrl(), CHANNEL_TARGET); // LDS -> {A, B}, version 2 // Failed to parse endpoint B @@ -802,6 +916,8 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti // {A} -> ACK, version 2 // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> does not exist + verifyResourceValidInvalidMetrics(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), + LDS.typeUrl()); List errorsV2 = ImmutableList.of("LDS response Listener 'B' validation error: "); verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 3); verifyResourceMetadataNacked( @@ -809,9 +925,25 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti errorsV2); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(LDS, "C"); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), + CHANNEL_TARGET); + verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", LDS.typeUrl(), + CHANNEL_TARGET); + verify(callbackMetricReporter).reportResourceCounts(1, "does_not_exist", LDS.typeUrl(), + CHANNEL_TARGET); + verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", RDS.typeUrl(), + CHANNEL_TARGET); } else { // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(2, "acked", LDS.typeUrl(), + CHANNEL_TARGET); + verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", LDS.typeUrl(), + CHANNEL_TARGET); + verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", RDS.typeUrl(), + CHANNEL_TARGET); } call.verifyRequestNack(LDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); // {A.1} -> version 1 @@ -822,6 +954,7 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti // Verify {C.1} stays in the previous version VERSION_1, no matter {C} is deleted or not. verifyResourceMetadataAcked(RDS, "C.1", resourcesV11.get("C.1"), VERSION_1, TIME_INCREMENT * 2); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -877,6 +1010,9 @@ public void wrappedLdsResource_preferWrappedName() { public void ldsResourceFound_containsRdsName() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), + CHANNEL_TARGET); call.sendResponse(LDS, testListenerRds, VERSION_1, "0000"); // Client sends an ACK LDS request. @@ -884,8 +1020,11 @@ public void ldsResourceFound_containsRdsName() { verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); verifyGoldenListenerRds(ldsUpdateCaptor.getValue()); assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerRds, VERSION_1, TIME_INCREMENT); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -899,7 +1038,7 @@ public void cachedLdsResource_data() { call.verifyRequest(LDS, LDS_RESOURCE, VERSION_1, "0000", NODE); ResourceWatcher watcher = mock(ResourceWatcher.class); - xdsClient.watchXdsResource(XdsListenerResource.getInstance(),LDS_RESOURCE, watcher); + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), LDS_RESOURCE, watcher); verify(watcher).onChanged(ldsUpdateCaptor.capture()); verifyGoldenListenerRds(ldsUpdateCaptor.getValue()); call.verifyNoMoreRequest(); @@ -916,7 +1055,7 @@ public void cachedLdsResource_absent() { verify(ldsResourceWatcher).onResourceDoesNotExist(LDS_RESOURCE); // Add another watcher. ResourceWatcher watcher = mock(ResourceWatcher.class); - xdsClient.watchXdsResource(XdsListenerResource.getInstance(),LDS_RESOURCE, watcher); + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), LDS_RESOURCE, watcher); verify(watcher).onResourceDoesNotExist(LDS_RESOURCE); call.verifyNoMoreRequest(); verifyResourceMetadataDoesNotExist(LDS, LDS_RESOURCE); @@ -928,6 +1067,10 @@ public void ldsResourceUpdated() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); verifyResourceMetadataRequested(LDS, LDS_RESOURCE); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), + CHANNEL_TARGET + ); // Initial LDS response. call.sendResponse(LDS, testListenerVhosts, VERSION_1, "0000"); @@ -935,6 +1078,8 @@ public void ldsResourceUpdated() { verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); verifyGoldenListenerVhosts(ldsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET); // Updated LDS response. call.sendResponse(LDS, testListenerRds, VERSION_2, "0001"); @@ -942,9 +1087,13 @@ public void ldsResourceUpdated() { verify(ldsResourceWatcher, times(2)).onChanged(ldsUpdateCaptor.capture()); verifyGoldenListenerRds(ldsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerRds, VERSION_2, TIME_INCREMENT * 2); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", LDS.typeUrl(), + CHANNEL_TARGET); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); assertThat(channelForCustomAuthority).isNull(); assertThat(channelForEmptyAuthority).isNull(); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -952,6 +1101,10 @@ public void cancelResourceWatcherNotRemoveUrlSubscribers() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); verifyResourceMetadataRequested(LDS, LDS_RESOURCE); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), + CHANNEL_TARGET + ); // Initial LDS response. call.sendResponse(LDS, testListenerVhosts, VERSION_1, "0000"); @@ -959,6 +1112,8 @@ public void cancelResourceWatcherNotRemoveUrlSubscribers() { verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); verifyGoldenListenerVhosts(ldsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET); xdsClient.watchXdsResource(XdsListenerResource.getInstance(), LDS_RESOURCE + "1", ldsResourceWatcher); @@ -974,6 +1129,11 @@ public void cancelResourceWatcherNotRemoveUrlSubscribers() { verifyGoldenListenerVhosts(ldsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts2, VERSION_2, TIME_INCREMENT * 2); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", LDS.typeUrl(), + CHANNEL_TARGET + ); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -1038,6 +1198,10 @@ public void ldsResourceUpdated_withXdstpResourceName_withWrongType() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), ldsResourceName, ldsResourceWatcher); assertThat(channelForCustomAuthority).isNotNull(); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), + CHANNEL_TARGET + ); String ldsResourceNameWithWrongType = "xdstp://authority.xds.com/envoy.config.route.v3.RouteConfiguration/listener1"; @@ -1049,6 +1213,11 @@ public void ldsResourceUpdated_withXdstpResourceName_withWrongType() { LDS, ldsResourceName, "", "0000", NODE, ImmutableList.of( "Unsupported resource name: " + ldsResourceNameWithWrongType + " for type: LDS")); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "requested", LDS.typeUrl(), + CHANNEL_TARGET + ); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -1109,7 +1278,7 @@ public void cdsResourceUpdated_withXdstpResourceName_withWrongType() { @Test public void cdsResourceUpdated_withXdstpResourceName_unknownAuthority() { String cdsResourceName = "xdstp://unknown.example.com/envoy.config.cluster.v3.Cluster/cluster1"; - xdsClient.watchXdsResource(XdsClusterResource.getInstance(),cdsResourceName, + xdsClient.watchXdsResource(XdsClusterResource.getInstance(), cdsResourceName, cdsResourceWatcher); verify(cdsResourceWatcher).onError(errorCaptor.capture()); Status error = errorCaptor.getValue(); @@ -1117,7 +1286,7 @@ public void cdsResourceUpdated_withXdstpResourceName_unknownAuthority() { assertThat(error.getDescription()).isEqualTo( "Wrong configuration: xds server does not exist for resource " + cdsResourceName); assertThat(resourceDiscoveryCalls.poll()).isNull(); - xdsClient.cancelXdsResourceWatch(XdsClusterResource.getInstance(),cdsResourceName, + xdsClient.cancelXdsResourceWatch(XdsClusterResource.getInstance(), cdsResourceName, cdsResourceWatcher); assertThat(resourceDiscoveryCalls.poll()).isNull(); } @@ -1166,6 +1335,10 @@ public void edsResourceUpdated_withXdstpResourceName_unknownAuthority() { public void ldsResourceUpdate_withFaultInjection() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), + CHANNEL_TARGET + ); Any listener = Any.pack( mf.buildListenerWithApiListener( LDS_RESOURCE, @@ -1203,6 +1376,9 @@ public void ldsResourceUpdate_withFaultInjection() { call.verifyRequest(LDS, LDS_RESOURCE, VERSION_1, "0000", NODE); verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, listener, VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); LdsUpdate ldsUpdate = ldsUpdateCaptor.getValue(); @@ -1225,6 +1401,7 @@ public void ldsResourceUpdate_withFaultInjection() { assertThat(faultConfig.faultAbort().percent().denominatorType()) .isEqualTo(DenominatorType.MILLION); assertThat(faultConfig.maxActiveFaults()).isEqualTo(101); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -1234,6 +1411,10 @@ public void ldsResourceDeleted() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); verifyResourceMetadataRequested(LDS, LDS_RESOURCE); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), + CHANNEL_TARGET + ); // Initial LDS response. call.sendResponse(LDS, testListenerVhosts, VERSION_1, "0000"); @@ -1241,6 +1422,9 @@ public void ldsResourceDeleted() { verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); verifyGoldenListenerVhosts(ldsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); // Empty LDS response deletes the listener. @@ -1248,13 +1432,18 @@ public void ldsResourceDeleted() { call.verifyRequest(LDS, LDS_RESOURCE, VERSION_2, "0001", NODE); verify(ldsResourceWatcher).onResourceDoesNotExist(LDS_RESOURCE); verifyResourceMetadataDoesNotExist(LDS, LDS_RESOURCE); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "does_not_exist", LDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); + verifyNoMoreInteractions(callbackMetricReporter); } /** * When ignore_resource_deletion server feature is on, xDS client should keep the deleted listener * on empty response, and resume the normal work when LDS contains the listener again. - * */ + */ @Test public void ldsResourceDeleted_ignoreResourceDeletion() { Assume.assumeTrue(ignoreResourceDeletion()); @@ -1262,6 +1451,10 @@ public void ldsResourceDeleted_ignoreResourceDeletion() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); verifyResourceMetadataRequested(LDS, LDS_RESOURCE); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), + CHANNEL_TARGET + ); // Initial LDS response. call.sendResponse(LDS, testListenerVhosts, VERSION_1, "0000"); @@ -1269,6 +1462,9 @@ public void ldsResourceDeleted_ignoreResourceDeletion() { verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); verifyGoldenListenerVhosts(ldsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); // Empty LDS response does not delete the listener. @@ -1276,6 +1472,10 @@ public void ldsResourceDeleted_ignoreResourceDeletion() { call.verifyRequest(LDS, LDS_RESOURCE, VERSION_2, "0001", NODE); // The resource is still ACKED at VERSION_1 (no changes). verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", LDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); // onResourceDoesNotExist not called verify(ldsResourceWatcher, never()).onResourceDoesNotExist(LDS_RESOURCE); @@ -1288,8 +1488,13 @@ public void ldsResourceDeleted_ignoreResourceDeletion() { // LDS is now ACKEd at VERSION_3. verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_3, TIME_INCREMENT * 3); + reportResourceCount(); + verify(callbackMetricReporter, times(3)).reportResourceCounts(1, "acked", LDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); verifyNoMoreInteractions(ldsResourceWatcher); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -1298,14 +1503,18 @@ public void multipleLdsWatchers() { String ldsResourceTwo = "bar.googleapis.com"; ResourceWatcher watcher1 = mock(ResourceWatcher.class); ResourceWatcher watcher2 = mock(ResourceWatcher.class); - xdsClient.watchXdsResource(XdsListenerResource.getInstance(),LDS_RESOURCE, ldsResourceWatcher); - xdsClient.watchXdsResource(XdsListenerResource.getInstance(),ldsResourceTwo, watcher1); - xdsClient.watchXdsResource(XdsListenerResource.getInstance(),ldsResourceTwo, watcher2); + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), ldsResourceTwo, watcher1); + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), ldsResourceTwo, watcher2); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); call.verifyRequest(LDS, ImmutableList.of(LDS_RESOURCE, ldsResourceTwo), "", "", NODE); // Both LDS resources were requested. verifyResourceMetadataRequested(LDS, LDS_RESOURCE); verifyResourceMetadataRequested(LDS, ldsResourceTwo); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(2, "requested", LDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(2, 0, 0, 0); fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); @@ -1314,6 +1523,10 @@ public void multipleLdsWatchers() { verify(watcher2).onResourceDoesNotExist(ldsResourceTwo); verifyResourceMetadataDoesNotExist(LDS, LDS_RESOURCE); verifyResourceMetadataDoesNotExist(LDS, ldsResourceTwo); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(2, "does_not_exist", LDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(2, 0, 0, 0); Any listenerTwo = Any.pack(mf.buildListenerWithApiListenerForRds(ldsResourceTwo, RDS_RESOURCE)); @@ -1332,7 +1545,11 @@ public void multipleLdsWatchers() { // Metadata of both listeners is stored. verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, ldsResourceTwo, listenerTwo, VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(2, "acked", LDS.typeUrl(), CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(2, 0, 0, 0); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -1340,7 +1557,7 @@ public void rdsResourceNotFound() { DiscoveryRpcCall call = startResourceWatcher(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, rdsResourceWatcher); Any routeConfig = Any.pack(mf.buildRouteConfiguration("route-bar.googleapis.com", - mf.buildOpaqueVirtualHosts(2))); + mf.buildOpaqueVirtualHosts(2))); call.sendResponse(RDS, routeConfig, VERSION_1, "0000"); // Client sends an ACK RDS request. @@ -1399,6 +1616,10 @@ public void rdsResponseErrorHandling_nackWeightedSumZero() { DiscoveryRpcCall call = startResourceWatcher(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, rdsResourceWatcher); verifyResourceMetadataRequested(RDS, RDS_RESOURCE); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", RDS.typeUrl(), + CHANNEL_TARGET + ); io.envoyproxy.envoy.config.route.v3.RouteAction routeAction = io.envoyproxy.envoy.config.route.v3.RouteAction.newBuilder() @@ -1433,29 +1654,38 @@ public void rdsResponseErrorHandling_nackWeightedSumZero() { + "RouteConfiguration contains invalid virtual host: Virtual host [do not care] " + "contains invalid route : Route [route-blade] contains invalid RouteAction: " + "Sum of cluster weights should be above 0."); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "nacked", RDS.typeUrl(), CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(0, 0, 1, 0); // The response is NACKed with the same error message. call.verifyRequestNack(RDS, RDS_RESOURCE, "", "0000", NODE, errors); verify(rdsResourceWatcher, never()).onChanged(any(RdsUpdate.class)); + verifyNoMoreInteractions(callbackMetricReporter); } /** * Tests a subscribed RDS resource transitioned to and from the invalid state. * - * @see - * A40-csds-support.md + * @see + * A40-csds-support.md */ @Test public void rdsResponseErrorHandling_subscribedResourceInvalid() { List subscribedResourceNames = ImmutableList.of("A", "B", "C"); - xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(),"A", rdsResourceWatcher); - xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(),"B", rdsResourceWatcher); - xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(),"C", rdsResourceWatcher); + xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), "A", rdsResourceWatcher); + xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), "B", rdsResourceWatcher); + xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), "C", rdsResourceWatcher); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); assertThat(call).isNotNull(); verifyResourceMetadataRequested(RDS, "A"); verifyResourceMetadataRequested(RDS, "B"); verifyResourceMetadataRequested(RDS, "C"); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(3, "requested", RDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(0, 0, 3, 0); // RDS -> {A, B, C}, version 1 @@ -1466,9 +1696,13 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { "C", Any.pack(mf.buildRouteConfiguration("C", vhostsV1))); call.sendResponse(RDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 + verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + RDS.typeUrl()); verifyResourceMetadataAcked(RDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(RDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(RDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(3, "acked", RDS.typeUrl(), CHANNEL_TARGET); call.verifyRequest(RDS, subscribedResourceNames, VERSION_1, "0000", NODE); // RDS -> {A, B}, version 2 @@ -1480,12 +1714,19 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { // {A} -> ACK, version 2 // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> ACK, version 1 + verifyResourceValidInvalidMetrics(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), + RDS.typeUrl()); List errorsV2 = ImmutableList.of("RDS response RouteConfiguration 'B' validation error: "); verifyResourceMetadataAcked(RDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataNacked(RDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, VERSION_2, TIME_INCREMENT * 2, errorsV2); verifyResourceMetadataAcked(RDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(2, "acked", RDS.typeUrl(), CHANNEL_TARGET); + verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", RDS.typeUrl(), + CHANNEL_TARGET + ); call.verifyRequestNack(RDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); // RDS -> {B, C} version 3 @@ -1496,11 +1737,18 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { call.sendResponse(RDS, resourcesV3.values().asList(), VERSION_3, "0002"); // {A} -> ACK, version 2 // {B, C} -> ACK, version 3 + verifyResourceValidInvalidMetrics(1, 2, 0, CHANNEL_TARGET, xdsServerInfo.target(), + RDS.typeUrl()); verifyResourceMetadataAcked(RDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataAcked(RDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); verifyResourceMetadataAcked(RDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", RDS.typeUrl(), + CHANNEL_TARGET + ); call.verifyRequest(RDS, subscribedResourceNames, VERSION_3, "0002", NODE); verifySubscribedResourcesMetadataSizes(0, 0, 3, 0); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -1544,7 +1792,7 @@ public void cachedRdsResource_data() { call.verifyRequest(RDS, RDS_RESOURCE, VERSION_1, "0000", NODE); ResourceWatcher watcher = mock(ResourceWatcher.class); - xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(),RDS_RESOURCE, watcher); + xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, watcher); verify(watcher).onChanged(rdsUpdateCaptor.capture()); verifyGoldenRouteConfig(rdsUpdateCaptor.getValue()); call.verifyNoMoreRequest(); @@ -1561,7 +1809,7 @@ public void cachedRdsResource_absent() { verify(rdsResourceWatcher).onResourceDoesNotExist(RDS_RESOURCE); // Add another watcher. ResourceWatcher watcher = mock(ResourceWatcher.class); - xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(),RDS_RESOURCE, watcher); + xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, watcher); verify(watcher).onResourceDoesNotExist(RDS_RESOURCE); call.verifyNoMoreRequest(); verifyResourceMetadataDoesNotExist(RDS, RDS_RESOURCE); @@ -1573,6 +1821,10 @@ public void rdsResourceUpdated() { DiscoveryRpcCall call = startResourceWatcher(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, rdsResourceWatcher); verifyResourceMetadataRequested(RDS, RDS_RESOURCE); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", RDS.typeUrl(), + CHANNEL_TARGET + ); // Initial RDS response. call.sendResponse(RDS, testRouteConfig, VERSION_1, "0000"); @@ -1580,6 +1832,8 @@ public void rdsResourceUpdated() { verify(rdsResourceWatcher).onChanged(rdsUpdateCaptor.capture()); verifyGoldenRouteConfig(rdsUpdateCaptor.getValue()); verifyResourceMetadataAcked(RDS, RDS_RESOURCE, testRouteConfig, VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", RDS.typeUrl(), CHANNEL_TARGET); // Updated RDS response. Any routeConfigUpdated = @@ -1592,17 +1846,29 @@ public void rdsResourceUpdated() { assertThat(rdsUpdateCaptor.getValue().virtualHosts).hasSize(4); verifyResourceMetadataAcked(RDS, RDS_RESOURCE, routeConfigUpdated, VERSION_2, TIME_INCREMENT * 2); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", RDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(0, 0, 1, 0); + verifyNoMoreInteractions(callbackMetricReporter); } @Test public void rdsResourceDeletedByLdsApiListener() { - xdsClient.watchXdsResource(XdsListenerResource.getInstance(),LDS_RESOURCE, + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); - xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(),RDS_RESOURCE, + xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, rdsResourceWatcher); verifyResourceMetadataRequested(LDS, LDS_RESOURCE); verifyResourceMetadataRequested(RDS, RDS_RESOURCE); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", RDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); @@ -1611,6 +1877,11 @@ public void rdsResourceDeletedByLdsApiListener() { verifyGoldenListenerRds(ldsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerRds, VERSION_1, TIME_INCREMENT); verifyResourceMetadataRequested(RDS, RDS_RESOURCE); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "requested", RDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); call.sendResponse(RDS, testRouteConfig, VERSION_1, "0000"); @@ -1618,6 +1889,12 @@ public void rdsResourceDeletedByLdsApiListener() { verifyGoldenRouteConfig(rdsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerRds, VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(RDS, RDS_RESOURCE, testRouteConfig, VERSION_1, TIME_INCREMENT * 2); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", LDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", RDS.typeUrl(), CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); // The Listener is getting replaced configured with an RDS name, to the one configured with @@ -1631,7 +1908,14 @@ public void rdsResourceDeletedByLdsApiListener() { verifyResourceMetadataAcked(RDS, RDS_RESOURCE, testRouteConfig, VERSION_1, TIME_INCREMENT * 2); verifyResourceMetadataAcked( LDS, LDS_RESOURCE, testListenerVhosts, VERSION_2, TIME_INCREMENT * 3); + reportResourceCount(); + verify(callbackMetricReporter, times(3)).reportResourceCounts(1, "acked", LDS.typeUrl(), + CHANNEL_TARGET); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", RDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -1642,6 +1926,13 @@ public void rdsResourcesDeletedByLdsTcpListener() { rdsResourceWatcher); verifyResourceMetadataRequested(LDS, LISTENER_RESOURCE); verifyResourceMetadataRequested(RDS, RDS_RESOURCE); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", RDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); Message hcmFilter = mf.buildHttpConnectionManagerFilter( @@ -1666,6 +1957,11 @@ public void rdsResourcesDeletedByLdsTcpListener() { assertThat(parsedFilterChain.httpConnectionManager().rdsName()).isEqualTo(RDS_RESOURCE); verifyResourceMetadataAcked(LDS, LISTENER_RESOURCE, packedListener, VERSION_1, TIME_INCREMENT); verifyResourceMetadataRequested(RDS, RDS_RESOURCE); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "requested", RDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); // Simulates receiving the requested RDS resource. @@ -1673,6 +1969,12 @@ public void rdsResourcesDeletedByLdsTcpListener() { verify(rdsResourceWatcher).onChanged(rdsUpdateCaptor.capture()); verifyGoldenRouteConfig(rdsUpdateCaptor.getValue()); verifyResourceMetadataAcked(RDS, RDS_RESOURCE, testRouteConfig, VERSION_1, TIME_INCREMENT * 2); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", LDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", RDS.typeUrl(), CHANNEL_TARGET + ); // Simulates receiving an updated version of the requested LDS resource as a TCP listener // with a filter chain containing inlined RouteConfiguration. @@ -1698,7 +2000,14 @@ public void rdsResourcesDeletedByLdsTcpListener() { verifyResourceMetadataAcked(RDS, RDS_RESOURCE, testRouteConfig, VERSION_1, TIME_INCREMENT * 2); verifyResourceMetadataAcked( LDS, LISTENER_RESOURCE, packedListener, VERSION_2, TIME_INCREMENT * 3); + reportResourceCount(); + verify(callbackMetricReporter, times(3)).reportResourceCounts(1, "acked", LDS.typeUrl(), + CHANNEL_TARGET); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", RDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -1814,8 +2123,9 @@ public void cdsResponseErrorHandling_someResourcesFailedUnpack() { /** * Tests a subscribed CDS resource transitioned to and from the invalid state. * - * @see - * A40-csds-support.md + * @see + * A40-csds-support.md */ @Test public void cdsResponseErrorHandling_subscribedResourceInvalid() { @@ -1828,6 +2138,10 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataRequested(CDS, "A"); verifyResourceMetadataRequested(CDS, "B"); verifyResourceMetadataRequested(CDS, "C"); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(3, "requested", CDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(0, 3, 0, 0); // CDS -> {A, B, C}, version 1 @@ -1843,9 +2157,14 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { ))); call.sendResponse(CDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 + verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + CDS.typeUrl()); verifyResourceMetadataAcked(CDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(CDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(CDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(3, "acked", CDS.typeUrl(), CHANNEL_TARGET + ); call.verifyRequest(CDS, subscribedResourceNames, VERSION_1, "0000", NODE); // CDS -> {A, B}, version 2 @@ -1859,15 +2178,29 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { // {A} -> ACK, version 2 // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> does not exist + verifyResourceValidInvalidMetrics(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), + CDS.typeUrl()); List errorsV2 = ImmutableList.of("CDS response Cluster 'B' validation error: "); verifyResourceMetadataAcked(CDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataNacked(CDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, VERSION_2, TIME_INCREMENT * 2, errorsV2); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(CDS, "C"); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", CDS.typeUrl(), + CHANNEL_TARGET); + verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", CDS.typeUrl(), + CHANNEL_TARGET); + verify(callbackMetricReporter).reportResourceCounts(1, "does_not_exist", CDS.typeUrl(), + CHANNEL_TARGET); } else { // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. verifyResourceMetadataAcked(CDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(2, "acked", CDS.typeUrl(), + CHANNEL_TARGET); + verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", CDS.typeUrl(), + CHANNEL_TARGET); } call.verifyRequestNack(CDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); @@ -1882,15 +2215,29 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { call.sendResponse(CDS, resourcesV3.values().asList(), VERSION_3, "0002"); // {A} -> does not exit // {B, C} -> ACK, version 3 + verifyResourceValidInvalidMetrics(1, 2, 0, CHANNEL_TARGET, xdsServerInfo.target(), + CDS.typeUrl()); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(CDS, "A"); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(2, "acked", CDS.typeUrl(), CHANNEL_TARGET + ); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "does_not_exist", + CDS.typeUrl(), CHANNEL_TARGET + ); } else { // When resource deletion is disabled, {A} stays ACKed in the previous version VERSION_2. verifyResourceMetadataAcked(CDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", CDS.typeUrl(), + CHANNEL_TARGET + ); } verifyResourceMetadataAcked(CDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); verifyResourceMetadataAcked(CDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); + call.verifyRequest(CDS, subscribedResourceNames, VERSION_3, "0002", NODE); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -1910,6 +2257,13 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti verifyResourceMetadataRequested(EDS, "A.1"); verifyResourceMetadataRequested(EDS, "B.1"); verifyResourceMetadataRequested(EDS, "C.1"); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(3, "requested", CDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(3, "requested", EDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(0, 3, 0, 3); // CDS -> {A, B, C}, version 1 @@ -1925,9 +2279,16 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti ))); call.sendResponse(CDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 + verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + CDS.typeUrl()); verifyResourceMetadataAcked(CDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(CDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(CDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(3, "acked", CDS.typeUrl(), CHANNEL_TARGET); + verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "requested", EDS.typeUrl(), + CHANNEL_TARGET + ); call.verifyRequest(CDS, subscribedResourceNames, VERSION_1, "0000", NODE); // EDS -> {A.1, B.1, C.1}, version 1 @@ -1939,9 +2300,17 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti "C.1", Any.pack(mf.buildClusterLoadAssignment("C.1", endpointsV1, dropOverloads))); call.sendResponse(EDS, resourcesV11.values().asList(), VERSION_1, "0000"); // {A.1, B.1, C.1} -> ACK, version 1 + verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + EDS.typeUrl()); verifyResourceMetadataAcked(EDS, "A.1", resourcesV11.get("A.1"), VERSION_1, TIME_INCREMENT * 2); verifyResourceMetadataAcked(EDS, "B.1", resourcesV11.get("B.1"), VERSION_1, TIME_INCREMENT * 2); verifyResourceMetadataAcked(EDS, "C.1", resourcesV11.get("C.1"), VERSION_1, TIME_INCREMENT * 2); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", CDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(3, "acked", EDS.typeUrl(), CHANNEL_TARGET + ); // CDS -> {A, B}, version 2 // Failed to parse endpoint B @@ -1954,6 +2323,8 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti // {A} -> ACK, version 2 // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> does not exist + verifyResourceValidInvalidMetrics(1, 1, 1, + CHANNEL_TARGET, xdsServerInfo.target(), CDS.typeUrl()); List errorsV2 = ImmutableList.of("CDS response Cluster 'B' validation error: "); verifyResourceMetadataAcked(CDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 3); verifyResourceMetadataNacked( @@ -1961,9 +2332,30 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti errorsV2); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(CDS, "C"); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", CDS.typeUrl(), CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", CDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(1, "does_not_exist", CDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", EDS.typeUrl(), + CHANNEL_TARGET + ); } else { // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. verifyResourceMetadataAcked(CDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(2, "acked", CDS.typeUrl(), CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", CDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", EDS.typeUrl(), + CHANNEL_TARGET + ); } call.verifyRequestNack(CDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); // {A.1} -> version 1 @@ -1974,6 +2366,7 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti // Verify {C.1} stays in the previous version VERSION_1. {C1} deleted or not does not matter. verifyResourceMetadataAcked(EDS, "C.1", resourcesV11.get("C.1"), VERSION_1, TIME_INCREMENT * 2); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -2144,7 +2537,7 @@ public void cdsResponseWithUpstreamTlsContext() { "envoy.transport_sockets.tls", null, null)); List clusters = ImmutableList.of( Any.pack(mf.buildLogicalDnsCluster("cluster-bar.googleapis.com", - "dns-service-bar.googleapis.com", 443, "round_robin", null, null,false, null, null)), + "dns-service-bar.googleapis.com", 443, "round_robin", null, null, false, null, null)), clusterEds, Any.pack(mf.buildEdsCluster("cluster-baz.googleapis.com", null, "round_robin", null, null, false, null, "envoy.transport_sockets.tls", null, null))); @@ -2176,7 +2569,7 @@ public void cdsResponseWithNewUpstreamTlsContext() { // Management server sends back CDS response with UpstreamTlsContext. Any clusterEds = Any.pack(mf.buildEdsCluster(CDS_RESOURCE, "eds-cluster-foo.googleapis.com", "round_robin", - null, null,true, + null, null, true, mf.buildNewUpstreamTlsContext("cert-instance-name", "cert1"), "envoy.transport_sockets.tls", null, null)); List clusters = ImmutableList.of( @@ -2207,6 +2600,10 @@ public void cdsResponseWithNewUpstreamTlsContext() { public void cdsResponseErrorHandling_badUpstreamTlsContext() { DiscoveryRpcCall call = startResourceWatcher(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", CDS.typeUrl(), + CHANNEL_TARGET + ); // Management server sends back CDS response with UpstreamTlsContext. List clusters = ImmutableList.of(Any @@ -2224,6 +2621,11 @@ public void cdsResponseErrorHandling_badUpstreamTlsContext() { call.verifyRequestNack(CDS, CDS_RESOURCE, "", "0000", NODE, ImmutableList.of(errorMsg)); verify(cdsResourceWatcher).onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, errorMsg); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "requested", CDS.typeUrl(), + CHANNEL_TARGET + ); + verifyNoMoreInteractions(callbackMetricReporter); } /** @@ -2234,6 +2636,10 @@ public void cdsResponseErrorHandling_badUpstreamTlsContext() { public void cdsResponseWithOutlierDetection() { DiscoveryRpcCall call = startResourceWatcher(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", CDS.typeUrl(), + CHANNEL_TARGET + ); OutlierDetection outlierDetectionXds = OutlierDetection.newBuilder() .setInterval(Durations.fromNanos(100)) @@ -2257,7 +2663,7 @@ public void cdsResponseWithOutlierDetection() { "envoy.transport_sockets.tls", null, outlierDetectionXds)); List clusters = ImmutableList.of( Any.pack(mf.buildLogicalDnsCluster("cluster-bar.googleapis.com", - "dns-service-bar.googleapis.com", 443, "round_robin", null, null,false, null, null)), + "dns-service-bar.googleapis.com", 443, "round_robin", null, null, false, null, null)), clusterEds, Any.pack(mf.buildEdsCluster("cluster-baz.googleapis.com", null, "round_robin", null, null, false, null, "envoy.transport_sockets.tls", null, outlierDetectionXds))); @@ -2292,7 +2698,11 @@ public void cdsResponseWithOutlierDetection() { assertThat(failurePercentageEjection.requestVolume()).isEqualTo(100); verifyResourceMetadataAcked(CDS, CDS_RESOURCE, clusterEds, VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", CDS.typeUrl(), CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(0, 1, 0, 0); + verifyNoMoreInteractions(callbackMetricReporter); } /** @@ -2304,6 +2714,10 @@ public void cdsResponseWithInvalidOutlierDetectionNacks() { DiscoveryRpcCall call = startResourceWatcher(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", CDS.typeUrl(), + CHANNEL_TARGET + ); OutlierDetection outlierDetectionXds = OutlierDetection.newBuilder() .setMaxEjectionPercent(UInt32Value.of(101)).build(); @@ -2316,7 +2730,7 @@ public void cdsResponseWithInvalidOutlierDetectionNacks() { "envoy.transport_sockets.tls", null, outlierDetectionXds)); List clusters = ImmutableList.of( Any.pack(mf.buildLogicalDnsCluster("cluster-bar.googleapis.com", - "dns-service-bar.googleapis.com", 443, "round_robin", null, null,false, null, null)), + "dns-service-bar.googleapis.com", 443, "round_robin", null, null, false, null, null)), clusterEds, Any.pack(mf.buildEdsCluster("cluster-baz.googleapis.com", null, "round_robin", null, null, false, null, "envoy.transport_sockets.tls", null, outlierDetectionXds))); @@ -2409,6 +2823,10 @@ public void validateOutlierDetection_enforcingFailurePercentageTooHigh() public void cdsResponseErrorHandling_badTransportSocketName() { DiscoveryRpcCall call = startResourceWatcher(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", CDS.typeUrl(), + CHANNEL_TARGET + ); // Management server sends back CDS response with UpstreamTlsContext. List clusters = ImmutableList.of(Any @@ -2422,6 +2840,10 @@ public void cdsResponseErrorHandling_badTransportSocketName() { String errorMsg = "CDS response Cluster 'cluster.googleapis.com' validation error: " + "transport-socket with name envoy.transport_sockets.bad not supported."; call.verifyRequestNack(CDS, CDS_RESOURCE, "", "0000", NODE, ImmutableList.of(errorMsg)); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "requested", CDS.typeUrl(), + CHANNEL_TARGET + ); verify(cdsResourceWatcher).onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, errorMsg); } @@ -2436,8 +2858,7 @@ public void cdsResponseErrorHandling_xdstpWithoutEdsConfig() { )); final Any okClusterRoundRobin = Any.pack(mf.buildEdsCluster(cdsResourceName, "eds-service-bar.googleapis.com", - "round_robin", null,null, false, null, "envoy.transport_sockets.tls", null, null)); - + "round_robin", null, null, false, null, "envoy.transport_sockets.tls", null, null)); DiscoveryRpcCall call = startResourceWatcher(XdsClusterResource.getInstance(), cdsResourceName, cdsResourceWatcher); @@ -2480,13 +2901,21 @@ public void cachedCdsResource_data() { public void cachedCdsResource_absent() { DiscoveryRpcCall call = startResourceWatcher(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", CDS.typeUrl(), + CHANNEL_TARGET + ); fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); verify(cdsResourceWatcher).onResourceDoesNotExist(CDS_RESOURCE); ResourceWatcher watcher = mock(ResourceWatcher.class); - xdsClient.watchXdsResource(XdsClusterResource.getInstance(),CDS_RESOURCE, watcher); + xdsClient.watchXdsResource(XdsClusterResource.getInstance(), CDS_RESOURCE, watcher); verify(watcher).onResourceDoesNotExist(CDS_RESOURCE); call.verifyNoMoreRequest(); verifyResourceMetadataDoesNotExist(CDS, CDS_RESOURCE); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "does_not_exist", CDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(0, 1, 0, 0); } @@ -2620,7 +3049,7 @@ public void cdsResourceDeleted() { /** * When ignore_resource_deletion server feature is on, xDS client should keep the deleted cluster * on empty response, and resume the normal work when CDS contains the cluster again. - * */ + */ @Test public void cdsResourceDeleted_ignoreResourceDeletion() { Assume.assumeTrue(ignoreResourceDeletion()); @@ -2666,9 +3095,9 @@ public void multipleCdsWatchers() { String cdsResourceTwo = "cluster-bar.googleapis.com"; ResourceWatcher watcher1 = mock(ResourceWatcher.class); ResourceWatcher watcher2 = mock(ResourceWatcher.class); - xdsClient.watchXdsResource(XdsClusterResource.getInstance(),CDS_RESOURCE, cdsResourceWatcher); - xdsClient.watchXdsResource(XdsClusterResource.getInstance(),cdsResourceTwo, watcher1); - xdsClient.watchXdsResource(XdsClusterResource.getInstance(),cdsResourceTwo, watcher2); + xdsClient.watchXdsResource(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); + xdsClient.watchXdsResource(XdsClusterResource.getInstance(), cdsResourceTwo, watcher1); + xdsClient.watchXdsResource(XdsClusterResource.getInstance(), cdsResourceTwo, watcher2); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); call.verifyRequest(CDS, Arrays.asList(CDS_RESOURCE, cdsResourceTwo), "", "", NODE); verifyResourceMetadataRequested(CDS, CDS_RESOURCE); @@ -2769,16 +3198,23 @@ public void edsCleanupNonceAfterUnsubscription() { DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); assertThat(call).isNotNull(); call.verifyRequest(EDS, "A.1", "", "", NODE); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", EDS.typeUrl(), + CHANNEL_TARGET + ); // EDS -> {A.1}, version 1 List dropOverloads = ImmutableList.of(); List endpointsV1 = ImmutableList.of(lbEndpointHealthy); ImmutableMap resourcesV1 = ImmutableMap.of( - "A.1", Any.pack(mf.buildClusterLoadAssignment("A.1", endpointsV1, dropOverloads))); + "A.1", Any.pack(mf.buildClusterLoadAssignment("A.1", endpointsV1, dropOverloads))); call.sendResponse(EDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A.1} -> ACK, version 1 call.verifyRequest(EDS, "A.1", VERSION_1, "0000", NODE); verify(edsResourceWatcher, times(1)).onChanged(any()); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", EDS.typeUrl(), CHANNEL_TARGET + ); // trigger an EDS resource unsubscription. xdsClient.cancelXdsResourceWatch(XdsEndpointResource.getInstance(), "A.1", edsResourceWatcher); @@ -2789,6 +3225,11 @@ public void edsCleanupNonceAfterUnsubscription() { // same as the initial request xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), "A.1", edsResourceWatcher); call.verifyRequest(EDS, "A.1", "", "", NODE, Mockito.timeout(2000).times(2)); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "requested", EDS.typeUrl(), + CHANNEL_TARGET + ); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -2835,20 +3276,25 @@ public void edsResponseErrorHandling_someResourcesFailedUnpack() { /** * Tests a subscribed EDS resource transitioned to and from the invalid state. * - * @see - * A40-csds-support.md + * @see + * A40-csds-support.md */ @Test public void edsResponseErrorHandling_subscribedResourceInvalid() { List subscribedResourceNames = ImmutableList.of("A", "B", "C"); - xdsClient.watchXdsResource(XdsEndpointResource.getInstance(),"A", edsResourceWatcher); - xdsClient.watchXdsResource(XdsEndpointResource.getInstance(),"B", edsResourceWatcher); - xdsClient.watchXdsResource(XdsEndpointResource.getInstance(),"C", edsResourceWatcher); + xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), "A", edsResourceWatcher); + xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), "B", edsResourceWatcher); + xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), "C", edsResourceWatcher); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); assertThat(call).isNotNull(); verifyResourceMetadataRequested(EDS, "A"); verifyResourceMetadataRequested(EDS, "B"); verifyResourceMetadataRequested(EDS, "C"); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(3, "requested", EDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(0, 0, 0, 3); // EDS -> {A, B, C}, version 1 @@ -2860,9 +3306,13 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { "C", Any.pack(mf.buildClusterLoadAssignment("C", endpointsV1, dropOverloads))); call.sendResponse(EDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 + verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + EDS.typeUrl()); verifyResourceMetadataAcked(EDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(EDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(EDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(3, "acked", EDS.typeUrl(), CHANNEL_TARGET); call.verifyRequest(EDS, subscribedResourceNames, VERSION_1, "0000", NODE); // EDS -> {A, B}, version 2 @@ -2875,12 +3325,19 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { // {A} -> ACK, version 2 // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> ACK, version 1 + verifyResourceValidInvalidMetrics(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), + EDS.typeUrl()); List errorsV2 = ImmutableList.of("EDS response ClusterLoadAssignment 'B' validation error: "); verifyResourceMetadataAcked(EDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataNacked(EDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, VERSION_2, TIME_INCREMENT * 2, errorsV2); verifyResourceMetadataAcked(EDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(2, "acked", EDS.typeUrl(), CHANNEL_TARGET); + verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", EDS.typeUrl(), + CHANNEL_TARGET + ); call.verifyRequestNack(EDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); // EDS -> {B, C} version 3 @@ -2892,11 +3349,17 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { call.sendResponse(EDS, resourcesV3.values().asList(), VERSION_3, "0002"); // {A} -> ACK, version 2 // {B, C} -> ACK, version 3 + verifyResourceValidInvalidMetrics(1, 2, 0, CHANNEL_TARGET, xdsServerInfo.target(), + EDS.typeUrl()); verifyResourceMetadataAcked(EDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataAcked(EDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); verifyResourceMetadataAcked(EDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", EDS.typeUrl(), + CHANNEL_TARGET); call.verifyRequest(EDS, subscribedResourceNames, VERSION_3, "0002", NODE); verifySubscribedResourcesMetadataSizes(0, 0, 0, 3); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -2940,7 +3403,7 @@ public void cachedEdsResource_data() { call.verifyRequest(EDS, EDS_RESOURCE, VERSION_1, "0000", NODE); // Add another watcher. ResourceWatcher watcher = mock(ResourceWatcher.class); - xdsClient.watchXdsResource(XdsEndpointResource.getInstance(),EDS_RESOURCE, watcher); + xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), EDS_RESOURCE, watcher); verify(watcher).onChanged(edsUpdateCaptor.capture()); validateGoldenClusterLoadAssignment(edsUpdateCaptor.getValue()); call.verifyNoMoreRequest(); @@ -2957,7 +3420,7 @@ public void cachedEdsResource_absent() { fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); verify(edsResourceWatcher).onResourceDoesNotExist(EDS_RESOURCE); ResourceWatcher watcher = mock(ResourceWatcher.class); - xdsClient.watchXdsResource(XdsEndpointResource.getInstance(),EDS_RESOURCE, watcher); + xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), EDS_RESOURCE, watcher); verify(watcher).onResourceDoesNotExist(EDS_RESOURCE); call.verifyNoMoreRequest(); verifyResourceMetadataDoesNotExist(EDS, EDS_RESOURCE); @@ -2976,6 +3439,10 @@ public void flowControlAbsent() throws Exception { anotherWatcher, fakeWatchClock.getScheduledExecutorService()); verifyResourceMetadataRequested(CDS, CDS_RESOURCE); verifyResourceMetadataRequested(CDS, anotherCdsResource); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(2, "requested", CDS.typeUrl(), + CHANNEL_TARGET + ); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); call.verifyRequest(CDS, Arrays.asList(CDS_RESOURCE, anotherCdsResource), "", "", NODE); @@ -2983,6 +3450,11 @@ public void flowControlAbsent() throws Exception { call.sendResponse(CDS, testClusterRoundRobin, VERSION_1, "0000"); verifyResourceMetadataAcked( CDS, CDS_RESOURCE, testClusterRoundRobin, VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", CDS.typeUrl(), CHANNEL_TARGET); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", CDS.typeUrl(), + CHANNEL_TARGET + ); call.verifyRequest(CDS, Arrays.asList(CDS_RESOURCE, anotherCdsResource), VERSION_1, "0000", NODE); verifyNoInteractions(cdsResourceWatcher, anotherWatcher); @@ -3010,6 +3482,12 @@ public void flowControlAbsent() throws Exception { assertThat(call.isReady()).isFalse(); verifyResourceMetadataAcked( CDS, CDS_RESOURCE, testClusterRoundRobin, VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", CDS.typeUrl(), + CHANNEL_TARGET); + verify(callbackMetricReporter).reportResourceCounts(1, "does_not_exist", CDS.typeUrl(), + CHANNEL_TARGET + ); barrier.await(); verify(cdsResourceWatcher, atLeastOnce()).onChanged(any()); String errorMsg = "CDS response Cluster 'cluster.googleapis.com2' validation error: " @@ -3021,6 +3499,12 @@ public void flowControlAbsent() throws Exception { verify(cdsResourceWatcher, times(2)).onChanged(any()); verify(anotherWatcher).onResourceDoesNotExist(eq(anotherCdsResource)); verify(anotherWatcher).onError(any()); + reportResourceCount(); + verify(callbackMetricReporter, times(3)).reportResourceCounts(1, "acked", CDS.typeUrl(), + CHANNEL_TARGET); + verify(callbackMetricReporter).reportResourceCounts(1, "nacked", CDS.typeUrl(), CHANNEL_TARGET + ); + verifyNoMoreInteractions(callbackMetricReporter); } private Answer blockUpdate(CyclicBarrier barrier) { @@ -3161,14 +3645,21 @@ public void edsResourceDeletedByCds() { String resource = "backend-service.googleapis.com"; ResourceWatcher cdsWatcher = mock(ResourceWatcher.class); ResourceWatcher edsWatcher = mock(ResourceWatcher.class); - xdsClient.watchXdsResource(XdsClusterResource.getInstance(),resource, cdsWatcher); - xdsClient.watchXdsResource(XdsEndpointResource.getInstance(),resource, edsWatcher); - xdsClient.watchXdsResource(XdsClusterResource.getInstance(),CDS_RESOURCE, cdsResourceWatcher); - xdsClient.watchXdsResource(XdsEndpointResource.getInstance(),EDS_RESOURCE, edsResourceWatcher); + xdsClient.watchXdsResource(XdsClusterResource.getInstance(), resource, cdsWatcher); + xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), resource, edsWatcher); + xdsClient.watchXdsResource(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); + xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), EDS_RESOURCE, edsResourceWatcher); verifyResourceMetadataRequested(CDS, CDS_RESOURCE); verifyResourceMetadataRequested(CDS, resource); verifyResourceMetadataRequested(EDS, EDS_RESOURCE); verifyResourceMetadataRequested(EDS, resource); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(2, "requested", CDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(2, "requested", EDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(0, 2, 0, 2); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); @@ -3191,6 +3682,12 @@ public void edsResourceDeletedByCds() { verifyResourceMetadataAcked(CDS, CDS_RESOURCE, clusters.get(1), VERSION_1, TIME_INCREMENT); verifyResourceMetadataRequested(EDS, EDS_RESOURCE); verifyResourceMetadataRequested(EDS, resource); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(2, "acked", CDS.typeUrl(), CHANNEL_TARGET + ); + verify(callbackMetricReporter, times(2)).reportResourceCounts(2, "requested", EDS.typeUrl(), + CHANNEL_TARGET + ); List clusterLoadAssignments = ImmutableList.of( @@ -3220,6 +3717,12 @@ public void edsResourceDeletedByCds() { // CDS not changed. verifyResourceMetadataAcked(CDS, resource, clusters.get(0), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(CDS, CDS_RESOURCE, clusters.get(1), VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(2, "acked", CDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(2, "acked", EDS.typeUrl(), CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(0, 2, 0, 2); clusters = ImmutableList.of( @@ -3242,7 +3745,15 @@ public void edsResourceDeletedByCds() { EDS, resource, clusterLoadAssignments.get(1), VERSION_1, TIME_INCREMENT * 2); // no change verifyResourceMetadataAcked(CDS, resource, clusters.get(0), VERSION_2, TIME_INCREMENT * 3); verifyResourceMetadataAcked(CDS, CDS_RESOURCE, clusters.get(1), VERSION_2, TIME_INCREMENT * 3); + reportResourceCount(); + verify(callbackMetricReporter, times(3)).reportResourceCounts(2, "acked", CDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter, times(2)).reportResourceCounts(2, "acked", EDS.typeUrl(), + CHANNEL_TARGET + ); verifySubscribedResourcesMetadataSizes(0, 2, 0, 2); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -3251,9 +3762,9 @@ public void multipleEdsWatchers() { String edsResourceTwo = "cluster-load-assignment-bar.googleapis.com"; ResourceWatcher watcher1 = mock(ResourceWatcher.class); ResourceWatcher watcher2 = mock(ResourceWatcher.class); - xdsClient.watchXdsResource(XdsEndpointResource.getInstance(),EDS_RESOURCE, edsResourceWatcher); - xdsClient.watchXdsResource(XdsEndpointResource.getInstance(),edsResourceTwo, watcher1); - xdsClient.watchXdsResource(XdsEndpointResource.getInstance(),edsResourceTwo, watcher2); + xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), EDS_RESOURCE, edsResourceWatcher); + xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), edsResourceTwo, watcher1); + xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), edsResourceTwo, watcher2); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); call.verifyRequest(EDS, Arrays.asList(EDS_RESOURCE, edsResourceTwo), "", "", NODE); verifyResourceMetadataRequested(EDS, EDS_RESOURCE); @@ -3338,12 +3849,18 @@ public void useIndependentRpcContext() { @Test public void streamClosedWithNoResponse() { - xdsClient.watchXdsResource(XdsListenerResource.getInstance(),LDS_RESOURCE, ldsResourceWatcher); - xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(),RDS_RESOURCE, + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); + xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, rdsResourceWatcher); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); + reportServerConnection(); + verify(callbackMetricReporter).reportServerConnections(1, CHANNEL_TARGET, + xdsServerInfo.target()); // Management server closes the RPC stream before sending any response. call.sendCompleted(); + reportServerConnection(); + verify(callbackMetricReporter).reportServerConnections(0, CHANNEL_TARGET, + xdsServerInfo.target()); verify(ldsResourceWatcher, Mockito.timeout(1000).times(1)) .onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, @@ -3351,24 +3868,34 @@ public void streamClosedWithNoResponse() { verify(rdsResourceWatcher).onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, "ADS stream closed with OK before receiving a response"); + verifyNoMoreInteractions(callbackMetricReporter); } @Test public void streamClosedAfterSendingResponses() { - xdsClient.watchXdsResource(XdsListenerResource.getInstance(),LDS_RESOURCE, ldsResourceWatcher); - xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(),RDS_RESOURCE, + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); + xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, rdsResourceWatcher); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); + reportServerConnection(); + verify(callbackMetricReporter).reportServerConnections(1, CHANNEL_TARGET, + xdsServerInfo.target()); ScheduledTask ldsResourceTimeout = Iterables.getOnlyElement(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); ScheduledTask rdsResourceTimeout = Iterables.getOnlyElement(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); call.sendResponse(LDS, testListenerRds, VERSION_1, "0000"); + reportServerConnection(); + verify(callbackMetricReporter, times(2)).reportServerConnections(1, CHANNEL_TARGET, + xdsServerInfo.target()); assertThat(ldsResourceTimeout.isCancelled()).isTrue(); call.sendResponse(RDS, testRouteConfig, VERSION_1, "0000"); assertThat(rdsResourceTimeout.isCancelled()).isTrue(); // Management server closes the RPC stream after sending responses. call.sendCompleted(); + reportServerConnection(); + verify(callbackMetricReporter, times(3)).reportServerConnections(1, CHANNEL_TARGET, + xdsServerInfo.target()); verify(ldsResourceWatcher, never()).onError(errorCaptor.capture()); verify(rdsResourceWatcher, never()).onError(errorCaptor.capture()); } @@ -3376,11 +3903,14 @@ public void streamClosedAfterSendingResponses() { @Test public void streamClosedAndRetryWithBackoff() { InOrder inOrder = Mockito.inOrder(backoffPolicyProvider, backoffPolicy1, backoffPolicy2); - xdsClient.watchXdsResource(XdsListenerResource.getInstance(),LDS_RESOURCE, ldsResourceWatcher); - xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(),RDS_RESOURCE, + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); + reportServerConnection(); + verify(callbackMetricReporter).reportServerConnections(1, CHANNEL_TARGET, + xdsServerInfo.target()); + xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, rdsResourceWatcher); - xdsClient.watchXdsResource(XdsClusterResource.getInstance(),CDS_RESOURCE, cdsResourceWatcher); - xdsClient.watchXdsResource(XdsEndpointResource.getInstance(),EDS_RESOURCE, edsResourceWatcher); + xdsClient.watchXdsResource(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); + xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), EDS_RESOURCE, edsResourceWatcher); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); call.verifyRequest(LDS, LDS_RESOURCE, "", "", NODE); call.verifyRequest(RDS, RDS_RESOURCE, "", "", NODE); @@ -3399,6 +3929,10 @@ public void streamClosedAndRetryWithBackoff() { verify(edsResourceWatcher).onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNKNOWN, ""); + reportServerConnection(); + verify(callbackMetricReporter).reportServerConnections(0, CHANNEL_TARGET, + xdsServerInfo.target()); + // Retry after backoff. inOrder.verify(backoffPolicyProvider).get(); inOrder.verify(backoffPolicy1).nextBackoffNanos(); @@ -3412,6 +3946,10 @@ public void streamClosedAndRetryWithBackoff() { call.verifyRequest(CDS, CDS_RESOURCE, "", "", NODE); call.verifyRequest(EDS, EDS_RESOURCE, "", "", NODE); + reportServerConnection(); + verify(callbackMetricReporter, times(2)).reportServerConnections(0, CHANNEL_TARGET, + xdsServerInfo.target()); + // Management server becomes unreachable. String errorMsg = "my fault"; call.sendError(Status.UNAVAILABLE.withDescription(errorMsg).asException()); @@ -3424,6 +3962,10 @@ public void streamClosedAndRetryWithBackoff() { verify(edsResourceWatcher, times(2)).onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, errorMsg); + reportServerConnection(); + verify(callbackMetricReporter, times(3)).reportServerConnections(0, CHANNEL_TARGET, + xdsServerInfo.target()); + // Retry after backoff. inOrder.verify(backoffPolicy1).nextBackoffNanos(); retryTask = @@ -3441,6 +3983,9 @@ public void streamClosedAndRetryWithBackoff() { mf.buildRouteConfiguration("do not care", mf.buildOpaqueVirtualHosts(2))))); call.sendResponse(LDS, listeners, "63", "3242"); call.verifyRequest(LDS, LDS_RESOURCE, "63", "3242", NODE); + reportServerConnection(); + verify(callbackMetricReporter, times(2)).reportServerConnections(1, CHANNEL_TARGET, + xdsServerInfo.target()); List routeConfigs = ImmutableList.of( Any.pack(mf.buildRouteConfiguration(RDS_RESOURCE, mf.buildOpaqueVirtualHosts(2)))); @@ -3455,6 +4000,10 @@ public void streamClosedAndRetryWithBackoff() { verify(edsResourceWatcher, times(2)).onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, errorMsg); + reportServerConnection(); + verify(callbackMetricReporter, times(3)).reportServerConnections(1, CHANNEL_TARGET, + xdsServerInfo.target()); + // Reset backoff sequence and retry after backoff. inOrder.verify(backoffPolicyProvider).get(); inOrder.verify(backoffPolicy2).nextBackoffNanos(); @@ -3477,6 +4026,10 @@ public void streamClosedAndRetryWithBackoff() { verify(edsResourceWatcher, times(3)).onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, ""); + reportServerConnection(); + verify(callbackMetricReporter, times(4)).reportServerConnections(0, CHANNEL_TARGET, + xdsServerInfo.target()); + // Retry after backoff. inOrder.verify(backoffPolicy2).nextBackoffNanos(); retryTask = @@ -3489,7 +4042,12 @@ public void streamClosedAndRetryWithBackoff() { call.verifyRequest(CDS, CDS_RESOURCE, "", "", NODE); call.verifyRequest(EDS, EDS_RESOURCE, "", "", NODE); + reportServerConnection(); + verify(callbackMetricReporter, times(5)).reportServerConnections(0, CHANNEL_TARGET, + xdsServerInfo.target()); + inOrder.verifyNoMoreInteractions(); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -3499,6 +4057,9 @@ public void streamClosedAndRetryRaceWithAddRemoveWatchers() { xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, rdsResourceWatcher); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); + reportServerConnection(); + verify(callbackMetricReporter).reportServerConnections(1, CHANNEL_TARGET, + xdsServerInfo.target()); call.sendError(Status.UNAVAILABLE.asException()); verify(ldsResourceWatcher, Mockito.timeout(1000).times(1)) .onError(errorCaptor.capture()); @@ -3509,6 +4070,10 @@ public void streamClosedAndRetryRaceWithAddRemoveWatchers() { Iterables.getOnlyElement(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)); assertThat(retryTask.getDelay(TimeUnit.NANOSECONDS)).isEqualTo(10L); + reportServerConnection(); + verify(callbackMetricReporter).reportServerConnections(0, CHANNEL_TARGET, + xdsServerInfo.target()); + xdsClient.cancelXdsResourceWatch(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); xdsClient.cancelXdsResourceWatch(XdsRouteConfigureResource.getInstance(), @@ -3523,12 +4088,21 @@ public void streamClosedAndRetryRaceWithAddRemoveWatchers() { call.verifyRequest(EDS, EDS_RESOURCE, "", "", NODE); call.verifyNoMoreRequest(); + reportServerConnection(); + verify(callbackMetricReporter, times(2)).reportServerConnections(0, CHANNEL_TARGET, + xdsServerInfo.target()); + call.sendResponse(LDS, testListenerRds, VERSION_1, "0000"); List routeConfigs = ImmutableList.of( Any.pack(mf.buildRouteConfiguration(RDS_RESOURCE, mf.buildOpaqueVirtualHosts(VHOST_SIZE)))); call.sendResponse(RDS, routeConfigs, VERSION_1, "0000"); + reportServerConnection(); + verify(callbackMetricReporter, times(2)).reportServerConnections(1, CHANNEL_TARGET, + xdsServerInfo.target()); + verifyNoMoreInteractions(ldsResourceWatcher, rdsResourceWatcher); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -3539,6 +4113,18 @@ public void streamClosedAndRetryRestartsResourceInitialFetchTimerForUnresolvedRe xdsClient.watchXdsResource(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), EDS_RESOURCE, edsResourceWatcher); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); + reportResourceCount(); + reportServerConnection(); + verify(callbackMetricReporter).reportServerConnections(1, CHANNEL_TARGET, + xdsServerInfo.target()); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), + CHANNEL_TARGET); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", RDS.typeUrl(), + CHANNEL_TARGET); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", CDS.typeUrl(), + CHANNEL_TARGET); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", EDS.typeUrl(), + CHANNEL_TARGET); ScheduledTask ldsResourceTimeout = Iterables.getOnlyElement(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); ScheduledTask rdsResourceTimeout = @@ -3549,9 +4135,34 @@ public void streamClosedAndRetryRestartsResourceInitialFetchTimerForUnresolvedRe Iterables.getOnlyElement(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); call.sendResponse(LDS, testListenerRds, VERSION_1, "0000"); assertThat(ldsResourceTimeout.isCancelled()).isTrue(); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET + ); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "requested", RDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "requested", CDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "requested", EDS.typeUrl(), + CHANNEL_TARGET + ); call.sendResponse(RDS, testRouteConfig, VERSION_1, "0000"); assertThat(rdsResourceTimeout.isCancelled()).isTrue(); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", LDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", RDS.typeUrl(), CHANNEL_TARGET + ); + verify(callbackMetricReporter, times(3)).reportResourceCounts(1, "requested", CDS.typeUrl(), + CHANNEL_TARGET); + verify(callbackMetricReporter, times(3)).reportResourceCounts(1, "requested", EDS.typeUrl(), + CHANNEL_TARGET); + reportServerConnection(); + verify(callbackMetricReporter, times(2)).reportServerConnections(1, CHANNEL_TARGET, + xdsServerInfo.target()); call.sendError(Status.UNAVAILABLE.asException()); assertThat(cdsResourceTimeout.isCancelled()).isTrue(); @@ -3560,12 +4171,29 @@ public void streamClosedAndRetryRestartsResourceInitialFetchTimerForUnresolvedRe verify(rdsResourceWatcher, never()).onError(errorCaptor.capture()); verify(cdsResourceWatcher, never()).onError(errorCaptor.capture()); verify(edsResourceWatcher, never()).onError(errorCaptor.capture()); + reportResourceCount(); + verify(callbackMetricReporter, times(3)).reportResourceCounts(1, "acked", LDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", RDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter, times(4)).reportResourceCounts(1, "requested", CDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter, times(4)).reportResourceCounts(1, "requested", EDS.typeUrl(), + CHANNEL_TARGET + ); + reportServerConnection(); + verify(callbackMetricReporter, times(3)).reportServerConnections(1, CHANNEL_TARGET, + xdsServerInfo.target()); fakeClock.forwardNanos(10L); assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(0); assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(0); assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); + verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -3578,13 +4206,13 @@ public void reportLoadStatsToServer() { lrsCall.sendResponse(Collections.singletonList(clusterName), 1000L); fakeClock.forwardNanos(1000L); - lrsCall.verifyNextReportClusters(Collections.singletonList(new String[] {clusterName, null})); + lrsCall.verifyNextReportClusters(Collections.singletonList(new String[]{clusterName, null})); dropStats.release(); fakeClock.forwardNanos(1000L); // In case of having unreported cluster stats, one last report will be sent after corresponding // stats object released. - lrsCall.verifyNextReportClusters(Collections.singletonList(new String[] {clusterName, null})); + lrsCall.verifyNextReportClusters(Collections.singletonList(new String[]{clusterName, null})); fakeClock.forwardNanos(1000L); // Currently load reporting continues (with empty stats) even if all stats objects have been @@ -3658,18 +4286,18 @@ public void serverSideListenerNotFound() { @Test public void serverSideListenerResponseErrorHandling_badDownstreamTlsContext() { GrpcXdsClientImplTestBase.DiscoveryRpcCall call = - startResourceWatcher(XdsListenerResource.getInstance(), LISTENER_RESOURCE, - ldsResourceWatcher); + startResourceWatcher(XdsListenerResource.getInstance(), LISTENER_RESOURCE, + ldsResourceWatcher); Message hcmFilter = mf.buildHttpConnectionManagerFilter( - "route-foo.googleapis.com", null, + "route-foo.googleapis.com", null, Collections.singletonList(mf.buildTerminalFilter())); Message downstreamTlsContext = CommonTlsContextTestsUtil.buildTestDownstreamTlsContext( - null, null,false); + null, null, false); Message filterChain = mf.buildFilterChain( - Collections.emptyList(), downstreamTlsContext, "envoy.transport_sockets.tls", + Collections.emptyList(), downstreamTlsContext, "envoy.transport_sockets.tls", hcmFilter); Message listener = - mf.buildListenerWithFilterChain(LISTENER_RESOURCE, 7000, "0.0.0.0", filterChain); + mf.buildListenerWithFilterChain(LISTENER_RESOURCE, 7000, "0.0.0.0", filterChain); List listeners = ImmutableList.of(Any.pack(listener)); call.sendResponse(LDS, listeners, "0", "0000"); // The response NACKed with errors indicating indices of the failed resources. @@ -3690,7 +4318,7 @@ public void serverSideListenerResponseErrorHandling_badTransportSocketName() { "route-foo.googleapis.com", null, Collections.singletonList(mf.buildTerminalFilter())); Message downstreamTlsContext = CommonTlsContextTestsUtil.buildTestDownstreamTlsContext( - "cert1", "cert2",false); + "cert1", "cert2", false); Message filterChain = mf.buildFilterChain( Collections.emptyList(), downstreamTlsContext, "envoy.transport_sockets.bad1", hcmFilter); @@ -3721,6 +4349,16 @@ public void sendingToStoppedServer() throws Exception { xdsClient.watchXdsResource(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); fakeClock.forwardTime(14, TimeUnit.SECONDS); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "unknown", LDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", CDS.typeUrl(), + CHANNEL_TARGET + ); + reportServerConnection(); + verify(callbackMetricReporter).reportServerConnections(0, CHANNEL_TARGET, + xdsServerInfo.target()); // Restart the server xdsServer = cleanupRule.register( @@ -3735,11 +4373,29 @@ public void sendingToStoppedServer() throws Exception { verify(ldsResourceWatcher, never()).onResourceDoesNotExist(LDS_RESOURCE); fakeClock.forwardTime(20, TimeUnit.SECONDS); // Trigger rpcRetryTimer DiscoveryRpcCall call = resourceDiscoveryCalls.poll(3, TimeUnit.SECONDS); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "unknown", LDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "requested", CDS.typeUrl(), + CHANNEL_TARGET + ); + reportServerConnection(); + verify(callbackMetricReporter, times(2)).reportServerConnections(0, CHANNEL_TARGET, + xdsServerInfo.target()); if (call == null) { // The first rpcRetry may have happened before the channel was ready fakeClock.forwardTime(50, TimeUnit.SECONDS); call = resourceDiscoveryCalls.poll(3, TimeUnit.SECONDS); } + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter, times(3)).reportResourceCounts(1, "requested", CDS.typeUrl(), + CHANNEL_TARGET + ); + // NOTE: There is a ScheduledExecutorService that may get involved due to the reconnect // so you cannot rely on the logic being single threaded. The timeout() in verifyRequest // is therefore necessary to avoid flakiness. @@ -3750,7 +4406,17 @@ public void sendingToStoppedServer() throws Exception { verifyGoldenListenerVhosts(ldsUpdateCaptor.getValue()); assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET + ); + verify(callbackMetricReporter, times(4)).reportResourceCounts(1, "requested", CDS.typeUrl(), + CHANNEL_TARGET + ); + reportServerConnection(); + verify(callbackMetricReporter).reportServerConnections(1, CHANNEL_TARGET, + xdsServerInfo.target()); verifySubscribedResourcesMetadataSizes(1, 1, 0, 0); + verifyNoMoreInteractions(callbackMetricReporter); } catch (Throwable t) { throw t; // This allows putting a breakpoint here for debugging } @@ -3782,6 +4448,265 @@ public void sendToNonexistentServer() throws Exception { client.shutdown(); } + @Test + public void validAndInvalidResourceMetricReport() { + xdsClient.watchXdsResource(XdsClusterResource.getInstance(), "A", cdsResourceWatcher); + xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), "A.1", edsResourceWatcher); + xdsClient.watchXdsResource(XdsClusterResource.getInstance(), "B", cdsResourceWatcher); + xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), "B.1", edsResourceWatcher); + xdsClient.watchXdsResource(XdsClusterResource.getInstance(), "C", cdsResourceWatcher); + xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), "C.1", edsResourceWatcher); + DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); + assertThat(call).isNotNull(); + + // CDS -> {A, B, C}, version 1 + ImmutableMap resourcesV1 = ImmutableMap.of( + "A", Any.pack(mf.buildEdsCluster("A", "A.1", "round_robin", null, null, false, null, + "envoy.transport_sockets.tls", null, null + )), + "B", Any.pack(mf.buildEdsCluster("B", "B.1", "round_robin", null, null, false, null, + "envoy.transport_sockets.tls", null, null + )), + "C", Any.pack(mf.buildEdsCluster("C", "C.1", "round_robin", null, null, false, null, + "envoy.transport_sockets.tls", null, null + ))); + call.sendResponse(CDS, resourcesV1.values().asList(), VERSION_1, "0000"); + // {A, B, C} -> ACK, version 1 + verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + CDS.typeUrl()); + + // EDS -> {A.1, B.1, C.1}, version 1 + List dropOverloads = ImmutableList.of(); + List endpointsV1 = ImmutableList.of(lbEndpointHealthy); + ImmutableMap resourcesV11 = ImmutableMap.of( + "A.1", Any.pack(mf.buildClusterLoadAssignment("A.1", endpointsV1, dropOverloads)), + "B.1", Any.pack(mf.buildClusterLoadAssignment("B.1", endpointsV1, dropOverloads)), + "C.1", Any.pack(mf.buildClusterLoadAssignment("C.1", endpointsV1, dropOverloads))); + call.sendResponse(EDS, resourcesV11.values().asList(), VERSION_1, "0000"); + // {A.1, B.1, C.1} -> ACK, version 1 + verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + EDS.typeUrl()); + + // CDS -> {A, B}, version 2 + // Failed to parse endpoint B + ImmutableMap resourcesV2 = ImmutableMap.of( + "A", Any.pack(mf.buildEdsCluster("A", "A.2", "round_robin", null, null, false, null, + "envoy.transport_sockets.tls", null, null + )), + "B", Any.pack(mf.buildClusterInvalid("B"))); + call.sendResponse(CDS, resourcesV2.values().asList(), VERSION_2, "0001"); + // {A} -> ACK, version 2 + // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B + // {C} -> does not exist + verifyResourceValidInvalidMetrics(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), + CDS.typeUrl()); + } + + @Test + public void serverFailureMetricReport() { + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); + xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, + rdsResourceWatcher); + DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); + // Management server closes the RPC stream before sending any response. + call.sendCompleted(); + verify(ldsResourceWatcher, Mockito.timeout(1000).times(1)) + .onError(errorCaptor.capture()); + verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, + "ADS stream closed with OK before receiving a response"); + verify(rdsResourceWatcher).onError(errorCaptor.capture()); + verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, + "ADS stream closed with OK before receiving a response"); + verify(xdsClientMetricReporter, times(1)).reportServerFailure(eq(1L), eq(CHANNEL_TARGET), eq( + xdsServerInfo.target())); + } + + @Test + public void serverFailureMetricReport_forRetryAndBackoff() { + InOrder inOrder = Mockito.inOrder(backoffPolicyProvider, backoffPolicy1, backoffPolicy2); + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); + xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, + rdsResourceWatcher); + xdsClient.watchXdsResource(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); + xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), EDS_RESOURCE, edsResourceWatcher); + DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); + + // Management server closes the RPC stream with an error. + call.sendError(Status.UNKNOWN.asException()); + verify(xdsClientMetricReporter, times(1)).reportServerFailure(eq(1L), eq(CHANNEL_TARGET), + eq(xdsServerInfo.target())); + + // Retry after backoff. + inOrder.verify(backoffPolicyProvider).get(); + inOrder.verify(backoffPolicy1).nextBackoffNanos(); + ScheduledTask retryTask = + Iterables.getOnlyElement(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)); + assertThat(retryTask.getDelay(TimeUnit.NANOSECONDS)).isEqualTo(10L); + fakeClock.forwardNanos(10L); + call = resourceDiscoveryCalls.poll(); + + // Management server becomes unreachable. + String errorMsg = "my fault"; + call.sendError(Status.UNAVAILABLE.withDescription(errorMsg).asException()); + verify(xdsClientMetricReporter, times(2)).reportServerFailure(eq(1L), eq(CHANNEL_TARGET), + eq(xdsServerInfo.target())); + + // Retry after backoff. + inOrder.verify(backoffPolicy1).nextBackoffNanos(); + retryTask = + Iterables.getOnlyElement(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)); + assertThat(retryTask.getDelay(TimeUnit.NANOSECONDS)).isEqualTo(100L); + fakeClock.forwardNanos(100L); + call = resourceDiscoveryCalls.poll(); + + List resources = ImmutableList.of(FAILING_ANY, testListenerRds, FAILING_ANY); + call.sendResponse(LDS, resources, "63", "3242"); + + List routeConfigs = ImmutableList.of(FAILING_ANY, testRouteConfig, FAILING_ANY); + call.sendResponse(RDS, routeConfigs, "5", "6764"); + + call.sendError(Status.DEADLINE_EXCEEDED.asException()); + // Server Failure metric will not be reported, as stream is closed with an error after receiving + // a response + verify(xdsClientMetricReporter, times(2)).reportServerFailure(eq(1L), eq(CHANNEL_TARGET), + eq(xdsServerInfo.target())); + + // Reset backoff sequence and retry after backoff. + inOrder.verify(backoffPolicyProvider).get(); + inOrder.verify(backoffPolicy2).nextBackoffNanos(); + retryTask = + Iterables.getOnlyElement(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)); + assertThat(retryTask.getDelay(TimeUnit.NANOSECONDS)).isEqualTo(20L); + fakeClock.forwardNanos(20L); + call = resourceDiscoveryCalls.poll(); + + // Management server becomes unreachable again. + call.sendError(Status.UNAVAILABLE.asException()); + verify(xdsClientMetricReporter, times(3)).reportServerFailure(eq(1L), eq(CHANNEL_TARGET), + eq(xdsServerInfo.target())); + + // Retry after backoff. + inOrder.verify(backoffPolicy2).nextBackoffNanos(); + retryTask = + Iterables.getOnlyElement(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)); + assertThat(retryTask.getDelay(TimeUnit.NANOSECONDS)).isEqualTo(200L); + fakeClock.forwardNanos(200L); + call = resourceDiscoveryCalls.poll(); + + List clusters = ImmutableList.of(FAILING_ANY, testClusterRoundRobin); + call.sendResponse(CDS, clusters, VERSION_1, "0000"); + call.sendCompleted(); + // Server Failure metric will not be reported once again, as stream is closed after receiving a + // response + verify(xdsClientMetricReporter, times(3)).reportServerFailure(eq(1L), eq(CHANNEL_TARGET), + eq(xdsServerInfo.target())); + } + + + @Test + public void testReportResourceCounts() { + List subscribedResourceNames = ImmutableList.of("A", "B", "C"); + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), "A", ldsResourceWatcher); + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), "B", ldsResourceWatcher); + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), "C", ldsResourceWatcher); + DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); + assertThat(call).isNotNull(); + verifyResourceMetadataRequested(LDS, "A"); + verifyResourceMetadataRequested(LDS, "B"); + verifyResourceMetadataRequested(LDS, "C"); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(3, "requested", LDS.typeUrl(), + CHANNEL_TARGET + ); + verifySubscribedResourcesMetadataSizes(3, 0, 0, 0); + + // LDS -> {A, B, C}, version 1 + ImmutableMap resourcesV1 = ImmutableMap.of( + "A", Any.pack(mf.buildListenerWithApiListenerForRds("A", "A.1")), + "B", Any.pack(mf.buildListenerWithApiListenerForRds("B", "B.1")), + "C", Any.pack(mf.buildListenerWithApiListenerForRds("C", "C.1"))); + call.sendResponse(LDS, resourcesV1.values().asList(), VERSION_1, "0000"); + // {A, B, C} -> ACK, version 1 + verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + LDS.typeUrl()); + verifyResourceMetadataAcked(LDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); + verifyResourceMetadataAcked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); + verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(3, "acked", LDS.typeUrl(), CHANNEL_TARGET + ); + call.verifyRequest(LDS, subscribedResourceNames, VERSION_1, "0000", NODE); + + // LDS -> {A, B}, version 2 + // Failed to parse endpoint B + ImmutableMap resourcesV2 = ImmutableMap.of( + "A", Any.pack(mf.buildListenerWithApiListenerForRds("A", "A.2")), + "B", Any.pack(mf.buildListenerWithApiListenerInvalid("B"))); + call.sendResponse(LDS, resourcesV2.values().asList(), VERSION_2, "0001"); + // {A} -> ACK, version 2 + // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B + // {C} -> does not exist + verifyResourceValidInvalidMetrics(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), + LDS.typeUrl()); + List errorsV2 = ImmutableList.of("LDS response Listener 'B' validation error: "); + verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); + verifyResourceMetadataNacked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, + VERSION_2, TIME_INCREMENT * 2, errorsV2); + if (!ignoreResourceDeletion()) { + verifyResourceMetadataDoesNotExist(LDS, "C"); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", LDS.typeUrl(), + CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(1, "does_not_exist", LDS.typeUrl(), + CHANNEL_TARGET + ); + } else { + // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. + verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(2, "acked", LDS.typeUrl(), CHANNEL_TARGET + ); + verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", LDS.typeUrl(), + CHANNEL_TARGET + ); + } + call.verifyRequestNack(LDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); + + // LDS -> {B, C} version 3 + ImmutableMap resourcesV3 = ImmutableMap.of( + "B", Any.pack(mf.buildListenerWithApiListenerForRds("B", "B.3")), + "C", Any.pack(mf.buildListenerWithApiListenerForRds("C", "C.3"))); + call.sendResponse(LDS, resourcesV3.values().asList(), VERSION_3, "0002"); + // {A} -> does not exist + // {B, C} -> ACK, version 3 + verifyResourceValidInvalidMetrics(1, 2, 0, CHANNEL_TARGET, xdsServerInfo.target(), + LDS.typeUrl()); + if (!ignoreResourceDeletion()) { + verifyResourceMetadataDoesNotExist(LDS, "A"); + reportResourceCount(); + verify(callbackMetricReporter).reportResourceCounts(2, "acked", LDS.typeUrl(), CHANNEL_TARGET + ); + verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "does_not_exist", + LDS.typeUrl(), CHANNEL_TARGET + ); + } else { + // When resource deletion is disabled, {A} stays ACKed in the previous version VERSION_2. + verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); + reportResourceCount(); + verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", LDS.typeUrl(), + CHANNEL_TARGET + ); + } + verifyResourceMetadataAcked(LDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); + verifyResourceMetadataAcked(LDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); + call.verifyRequest(LDS, subscribedResourceNames, VERSION_3, "0002", NODE); + verifySubscribedResourcesMetadataSizes(3, 0, 0, 0); + verifyNoMoreInteractions(callbackMetricReporter); + } + private XdsClientImpl createXdsClient(String serverUri) { BootstrapInfo bootstrapInfo = buildBootStrap(serverUri); return new XdsClientImpl( @@ -3792,10 +4717,12 @@ private XdsClientImpl createXdsClient(String serverUri) { fakeClock.getStopwatchSupplier(), timeProvider, MessagePrinter.INSTANCE, - new TlsContextManagerImpl(bootstrapInfo)); + new TlsContextManagerImpl(bootstrapInfo), + CHANNEL_TARGET, + xdsClientMetricReporter); } - private BootstrapInfo buildBootStrap(String serverUri) { + private BootstrapInfo buildBootStrap(String serverUri) { ServerInfo xdsServerInfo = ServerInfo.create(serverUri, CHANNEL_CREDENTIALS, ignoreResourceDeletion(), true); @@ -3902,7 +4829,7 @@ protected void sendResponse( } protected void sendResponse(XdsResourceType type, Any resource, String versionInfo, - String nonce) { + String nonce) { sendResponse(type, ImmutableList.of(resource), versionInfo, nonce); } @@ -3934,6 +4861,7 @@ protected void sendResponse(List clusters, long loadReportIntervalNano) } protected abstract static class MessageFactory { + /** Throws {@link InvalidProtocolBufferException} on {@link Any#unpack(Class)}. */ protected static final Any FAILING_ANY = Any.newBuilder().setTypeUrl("fake").build(); diff --git a/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java b/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java index ee164938b2d..2e292f479e1 100644 --- a/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java @@ -23,6 +23,7 @@ import static org.mockito.Mockito.when; import io.grpc.InsecureChannelCredentials; +import io.grpc.MetricRecorder; import io.grpc.internal.ObjectPool; import io.grpc.xds.SharedXdsClientPoolProvider.RefCountedXdsClientObjectPool; import io.grpc.xds.client.Bootstrapper.BootstrapInfo; @@ -91,7 +92,7 @@ public void refCountedXdsClientObjectPool_delayedCreation() { BootstrapInfo bootstrapInfo = BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build(); RefCountedXdsClientObjectPool xdsClientPool = - new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET); + new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET, new MetricRecorder() {}); assertThat(xdsClientPool.getXdsClientForTest()).isNull(); XdsClient xdsClient = xdsClientPool.getObject(); assertThat(xdsClientPool.getXdsClientForTest()).isNotNull(); @@ -104,7 +105,7 @@ public void refCountedXdsClientObjectPool_refCounted() { BootstrapInfo bootstrapInfo = BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build(); RefCountedXdsClientObjectPool xdsClientPool = - new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET); + new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET, new MetricRecorder() {}); // getObject once XdsClient xdsClient = xdsClientPool.getObject(); assertThat(xdsClient).isNotNull(); @@ -124,7 +125,7 @@ public void refCountedXdsClientObjectPool_getObjectCreatesNewInstanceIfAlreadySh BootstrapInfo bootstrapInfo = BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build(); RefCountedXdsClientObjectPool xdsClientPool = - new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET); + new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET, new MetricRecorder() {}); XdsClient xdsClient1 = xdsClientPool.getObject(); assertThat(xdsClientPool.returnObject(xdsClient1)).isNull(); assertThat(xdsClient1.isShutDown()).isTrue(); diff --git a/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java b/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java new file mode 100644 index 00000000000..a42b0129db0 --- /dev/null +++ b/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java @@ -0,0 +1,219 @@ +/* + * Copyright 2024 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.SettableFuture; +import io.grpc.MetricInstrument; +import io.grpc.MetricRecorder; +import io.grpc.MetricRecorder.BatchCallback; +import io.grpc.MetricRecorder.BatchRecorder; +import io.grpc.MetricRecorder.Registration; +import io.grpc.testing.GrpcCleanupRule; +import io.grpc.xds.XdsClientMetricReporter.CallbackMetricReporter; +import io.grpc.xds.client.XdsClient; +import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatcher; +import org.mockito.Captor; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** + * Unit tests for {@link XdsClientMetricReporterImpl}. + */ +@RunWith(JUnit4.class) +public class XdsClientMetricReporterImplTest { + + private static final String target = "test-target"; + private static final String server = "trafficdirector.googleapis.com"; + private static final String resourceTypeUrl = + "resourceTypeUrl.googleapis.com/envoy.config.cluster.v3.Cluster"; + + @Rule + public final MockitoRule mocks = MockitoJUnit.rule(); + + @Mock + private MetricRecorder mockMetricRecorder; + @Mock + private XdsClient mockXdsClient; + @Mock + private BatchRecorder mockBatchRecorder; + @Mock + private Registration mockGaugeRegistration; + @Captor + private ArgumentCaptor gaugeBatchCallbackCaptor; + + private XdsClientMetricReporterImpl reporter; + + @Before + public void setUp() { + when(mockMetricRecorder.registerBatchCallback(any(), any())).thenReturn(mockGaugeRegistration); + reporter = new XdsClientMetricReporterImpl(mockMetricRecorder); + } + + @Test + public void reportResourceUpdates() { + reporter.reportResourceUpdates(10, 5, target, server, resourceTypeUrl); + verifyValidInvalidResourceCounterAdd(10, 5, target, server, resourceTypeUrl); + } + + @Test + public void reportServerFailure() { + reporter.reportServerFailure(1, target, server); + verifyServerFailureCounterAdd("grpc.xds_client.server_failure", 1, target, server); + } + + @Test + public void setXdsClient_reportMetrics() throws Exception { + SettableFuture future = SettableFuture.create(); + future.set(null); + when(mockXdsClient.reportResourceCounts(any(CallbackMetricReporter.class))) + .thenReturn(future); + when(mockXdsClient.reportServerConnections(any(CallbackMetricReporter.class))) + .thenReturn(future); + reporter.setXdsClient(mockXdsClient); + verify(mockMetricRecorder).registerBatchCallback(any(MetricRecorder.BatchCallback.class), + eqMetricInstrumentName("grpc.xds_client.connected"), + eqMetricInstrumentName("grpc.xds_client.resources")); + + reporter.reportCallbackMetrics(mockBatchRecorder); + verify(mockXdsClient).reportResourceCounts(any(CallbackMetricReporter.class)); + verify(mockXdsClient).reportServerConnections(any(CallbackMetricReporter.class)); + } + + @Test + public void setXdsClient_reportMetrics_exception() throws Exception { + SettableFuture future = SettableFuture.create(); + future.setException(new Exception("test")); + when(mockXdsClient.reportResourceCounts(any(CallbackMetricReporter.class))) + .thenReturn(future); + reporter.setXdsClient(mockXdsClient); + verify(mockMetricRecorder).registerBatchCallback(any(MetricRecorder.BatchCallback.class), + eqMetricInstrumentName("grpc.xds_client.connected"), + eqMetricInstrumentName("grpc.xds_client.resources")); + + reporter.reportCallbackMetrics(mockBatchRecorder); + verify(mockXdsClient).reportResourceCounts(any(CallbackMetricReporter.class)); + verify(mockXdsClient, never()).reportServerConnections( + any(CallbackMetricReporter.class)); + } + + // @Test + // public void metricGauges() throws ExecutionException, InterruptedException, TimeoutException { + // reporter.setXdsClient(mockXdsClient); + // verify(mockMetricRecorder).registerBatchCallback(gaugeBatchCallbackCaptor.capture(), + // any()); + // BatchCallback gaugeBatchCallback = gaugeBatchCallbackCaptor.getValue(); + // // Verify the correct resource gauge values when requested at this point. + // InOrder inOrder = inOrder(mockBatchRecorder); + // gaugeBatchCallback.accept(mockBatchRecorder); + // + // verify(mockXdsClient).reportResourceCounts(any(CallbackMetricReporter.class)); + // verify(mockXdsClient).reportServerConnections(any(CallbackMetricReporter.class)); + // + // inOrder.verify(mockBatchRecorder).recordLongGauge( + // argThat(new LongGaugeInstrumentArgumentMatcher("grpc.lb.rls.cache_entries")), eq(0L), + // any(), any()); + // inOrder.verify(mockBatchRecorder) + // .recordLongGauge(argThat(new LongGaugeInstrumentArgumentMatcher( + // "grpc.lb.rls.cache_size")), + // eq(0L), any(), any()); + // } + + @Test + public void close() { + reporter.setXdsClient(mockXdsClient); + verify(mockMetricRecorder).registerBatchCallback(any(MetricRecorder.BatchCallback.class), + eqMetricInstrumentName("grpc.xds_client.connected"), + eqMetricInstrumentName("grpc.xds_client.resources")); + reporter.close(); + } + + @Test + public void callbackMetricReporter() { + XdsClientMetricReporterImpl.CallbackMetricReporterImpl callback = + new XdsClientMetricReporterImpl.CallbackMetricReporterImpl(mockBatchRecorder); + + callback.reportServerConnections(1, target, server); + verify(mockBatchRecorder, times(1)).recordLongGauge( + eqMetricInstrumentName("grpc.xds_client.connected"), eq(1L), + eq(Lists.newArrayList(target, server)), + eq(Lists.newArrayList())); + + String cacheState = "requested"; + callback.reportResourceCounts(10, cacheState, resourceTypeUrl, target); + verify(mockBatchRecorder, times(1)).recordLongGauge( + eqMetricInstrumentName("grpc.xds_client.resources"), eq(10L), + eq(Arrays.asList(target, cacheState, resourceTypeUrl)), + eq(Collections.emptyList())); + } + + private void verifyServerFailureCounterAdd(String name, long value, + String dataPlaneTargetLabel, String xdsServer) { + verify(mockMetricRecorder).addLongCounter( + eqMetricInstrumentName(name), eq(value), + eq(Lists.newArrayList(dataPlaneTargetLabel, xdsServer)), + eq(Lists.newArrayList())); + } + + private void verifyValidInvalidResourceCounterAdd(long validResourceCount, + long invalidResourceCount, + String target, String xdsServer, String resourceTypeUrl) { + // TODO(dnvindhya): support the "authority" label once available. + verify(mockMetricRecorder).addLongCounter( + eqMetricInstrumentName("grpc.xds_client.resource_updates_valid"), eq(validResourceCount), + eq(Lists.newArrayList(target, xdsServer, resourceTypeUrl)), + eq(Lists.newArrayList())); + // TODO(dnvindhya): support the "authority" label once available. + verify(mockMetricRecorder).addLongCounter( + eqMetricInstrumentName("grpc.xds_client.resource_updates_invalid"), + eq(invalidResourceCount), + eq(Lists.newArrayList(target, xdsServer, resourceTypeUrl)), + eq(Lists.newArrayList())); + } + + @SuppressWarnings("TypeParameterUnusedInFormals") + private T eqMetricInstrumentName(String name) { + return argThat(new ArgumentMatcher() { + @Override + public boolean matches(T instrument) { + return instrument.getName().equals(name); + } + }); + } +} diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverProviderTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverProviderTest.java index a216c3de028..33aae268c60 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverProviderTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap; import io.grpc.ChannelLogger; import io.grpc.InternalServiceProviders; +import io.grpc.MetricRecorder; import io.grpc.NameResolver; import io.grpc.NameResolver.ServiceConfigParser; import io.grpc.NameResolverProvider; @@ -57,6 +58,7 @@ public void uncaughtException(Thread t, Throwable e) { .setServiceConfigParser(mock(ServiceConfigParser.class)) .setScheduledExecutorService(fakeClock.getScheduledExecutorService()) .setChannelLogger(mock(ChannelLogger.class)) + .setMetricRecorder(mock(MetricRecorder.class)) .build(); private XdsNameResolverProvider provider = new XdsNameResolverProvider(); diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java index f32c198f21f..23c42d85409 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java @@ -55,6 +55,7 @@ import io.grpc.Metadata; import io.grpc.MethodDescriptor; import io.grpc.MethodDescriptor.MethodType; +import io.grpc.MetricRecorder; import io.grpc.NameResolver; import io.grpc.NameResolver.ConfigOrError; import io.grpc.NameResolver.ResolutionResult; @@ -187,7 +188,7 @@ public void setUp() { RouterFilter.INSTANCE); resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, filterRegistry, null); + xdsClientPoolFactory, mockRandom, filterRegistry, null, new MetricRecorder() {}); } @After @@ -219,6 +220,12 @@ public ObjectPool getOrCreate(String target) throws XdsInitialization throw new XdsInitializationException("Fail to read bootstrap file"); } + @Override + public ObjectPool getOrCreate(String target, MetricRecorder metricRecorder) + throws XdsInitializationException { + throw new XdsInitializationException("Fail to read bootstrap file"); + } + @Override public List getTargets() { return null; @@ -227,7 +234,8 @@ public List getTargets() { resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null); + xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, + new MetricRecorder() {}); resolver.start(mockListener); verify(mockListener).onError(errorCaptor.capture()); Status error = errorCaptor.getValue(); @@ -240,7 +248,8 @@ public List getTargets() { public void resolving_withTargetAuthorityNotFound() { resolver = new XdsNameResolver(targetUri, "notfound.google.com", AUTHORITY, null, serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null); + xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, + new MetricRecorder() {}); resolver.start(mockListener); verify(mockListener).onError(errorCaptor.capture()); Status error = errorCaptor.getValue(); @@ -262,7 +271,7 @@ public void resolving_noTargetAuthority_templateWithoutXdstp() { resolver = new XdsNameResolver( targetUri, null, serviceAuthority, null, serviceConfigParser, syncContext, scheduler, xdsClientPoolFactory, - mockRandom, FilterRegistry.getDefaultRegistry(), null); + mockRandom, FilterRegistry.getDefaultRegistry(), null, new MetricRecorder() {}); resolver.start(mockListener); verify(mockListener, never()).onError(any(Status.class)); } @@ -282,7 +291,8 @@ public void resolving_noTargetAuthority_templateWithXdstp() { + "%5B::FFFF:129.144.52.38%5D:80?id=1"; resolver = new XdsNameResolver( targetUri, null, serviceAuthority, null, serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null); + xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, + new MetricRecorder() {}); resolver.start(mockListener); verify(mockListener, never()).onError(any(Status.class)); } @@ -302,7 +312,8 @@ public void resolving_noTargetAuthority_xdstpWithMultipleSlashes() { + "path/to/service?id=1"; resolver = new XdsNameResolver( targetUri, null, serviceAuthority, null, serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null); + xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, + new MetricRecorder() {}); // The Service Authority must be URL encoded, but unlike the LDS resource name. @@ -330,7 +341,8 @@ public void resolving_targetAuthorityInAuthoritiesMap() { + "%5B::FFFF:129.144.52.38%5D:80?bar=2&foo=1"; // query param canonified resolver = new XdsNameResolver(targetUri, "xds.authority.com", serviceAuthority, null, serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null); + xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, + new MetricRecorder() {}); resolver.start(mockListener); verify(mockListener, never()).onError(any(Status.class)); } @@ -362,7 +374,8 @@ public void resolving_ldsResourceUpdateRdsName() { .build(); resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null); + xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, + new MetricRecorder() {}); // use different ldsResourceName and service authority. The virtualhost lookup should use // service authority. expectedLdsResourceName = "test-" + expectedLdsResourceName; @@ -543,7 +556,8 @@ public void resolving_matchingVirtualHostNotFound_matchingOverrideAuthority() { resolver = new XdsNameResolver(targetUri, null, AUTHORITY, "random", serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null); + xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, + new MetricRecorder() {}); resolver.start(mockListener); FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); xdsClient.deliverLdsUpdate(0L, Arrays.asList(virtualHost)); @@ -566,7 +580,8 @@ public void resolving_matchingVirtualHostNotFound_notMatchingOverrideAuthority() resolver = new XdsNameResolver(targetUri, null, AUTHORITY, "random", serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null); + xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, + new MetricRecorder() {}); resolver.start(mockListener); FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); xdsClient.deliverLdsUpdate(0L, Arrays.asList(virtualHost)); @@ -577,7 +592,8 @@ public void resolving_matchingVirtualHostNotFound_notMatchingOverrideAuthority() public void resolving_matchingVirtualHostNotFoundForOverrideAuthority() { resolver = new XdsNameResolver(targetUri, null, AUTHORITY, AUTHORITY, serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null); + xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, + new MetricRecorder() {}); resolver.start(mockListener); FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); xdsClient.deliverLdsUpdate(0L, buildUnmatchedVirtualHosts()); @@ -661,7 +677,8 @@ public void retryPolicyInPerMethodConfigGeneratedByResolverIsValid() { ServiceConfigParser realParser = new ScParser( true, 5, 5, new AutoConfiguredLoadBalancerFactory("pick-first")); resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, realParser, syncContext, - scheduler, xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null); + scheduler, xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, + new MetricRecorder() {}); resolver.start(mockListener); FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); RetryPolicy retryPolicy = RetryPolicy.create( @@ -871,7 +888,8 @@ public void resolved_rpcHashingByChannelId() { when(mockRandom.nextLong()).thenReturn(123L); resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null); + xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, + new MetricRecorder() {}); resolver.start(mockListener); xdsClient = (FakeXdsClient) resolver.getXdsClient(); xdsClient.deliverLdsUpdate( @@ -1995,6 +2013,12 @@ public ObjectPool get(String target) { @Override public ObjectPool getOrCreate(String target) throws XdsInitializationException { + return getOrCreate(target, new MetricRecorder() {}); + } + + @Override + public ObjectPool getOrCreate(String target, MetricRecorder metricRecorder) + throws XdsInitializationException { targets.add(target); return new ObjectPool() { @Override diff --git a/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java b/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java index 791318c5355..6d80fc79bce 100644 --- a/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java +++ b/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.SettableFuture; import io.grpc.InsecureChannelCredentials; +import io.grpc.MetricRecorder; import io.grpc.internal.ObjectPool; import io.grpc.xds.EnvoyServerProtoData.ConnectionSourceType; import io.grpc.xds.EnvoyServerProtoData.FilterChain; @@ -152,6 +153,12 @@ public ObjectPool get(String target) { @Override public ObjectPool getOrCreate(String target) throws XdsInitializationException { + return getOrCreate(target, new MetricRecorder() {}); + } + + @Override + public ObjectPool getOrCreate(String target, MetricRecorder metricRecorder) + throws XdsInitializationException { return new ObjectPool() { @Override public XdsClient getObject() { From ef7bfcd8ebe2233a785bb7cfcb8c08f6bb1218b8 Mon Sep 17 00:00:00 2001 From: Vindhya Ningegowda Date: Fri, 1 Nov 2024 17:50:13 -0700 Subject: [PATCH 02/13] Rebase #2 --- api/src/main/java/io/grpc/NameResolver.java | 2 - .../test/java/io/grpc/NameResolverTest.java | 4 + .../grpc/internal/ManagedChannelImplTest.java | 25 + .../grpc/xds/SharedXdsClientPoolProvider.java | 5 + .../grpc/xds/XdsClientMetricReporterImpl.java | 24 +- .../java/io/grpc/xds/XdsNameResolver.java | 4 +- .../io/grpc/xds/client/XdsClientImpl.java | 6 +- .../grpc/xds/GrpcXdsClientImplTestBase.java | 1195 ++++++++--------- .../xds/SharedXdsClientPoolProviderTest.java | 12 +- .../xds/XdsClientMetricReporterImplTest.java | 138 +- .../java/io/grpc/xds/XdsNameResolverTest.java | 29 +- 11 files changed, 692 insertions(+), 752 deletions(-) diff --git a/api/src/main/java/io/grpc/NameResolver.java b/api/src/main/java/io/grpc/NameResolver.java index 58d49f559b0..8ed28730015 100644 --- a/api/src/main/java/io/grpc/NameResolver.java +++ b/api/src/main/java/io/grpc/NameResolver.java @@ -564,8 +564,6 @@ public Builder setOverrideAuthority(String authority) { /** * See {@link Args#getMetricRecorder()}. This is an optional field. - * - * @since 1.67.0 */ @ExperimentalApi("Insert github issue") public Builder setMetricRecorder(MetricRecorder metricRecorder) { diff --git a/api/src/test/java/io/grpc/NameResolverTest.java b/api/src/test/java/io/grpc/NameResolverTest.java index 1bc32ee7b1d..1a7c59f8df5 100644 --- a/api/src/test/java/io/grpc/NameResolverTest.java +++ b/api/src/test/java/io/grpc/NameResolverTest.java @@ -64,6 +64,7 @@ public class NameResolverTest { private final ChannelLogger channelLogger = mock(ChannelLogger.class); private final Executor executor = Executors.newSingleThreadExecutor(); private final String overrideAuthority = "grpc.io"; + private final MetricRecorder metricRecorder = new MetricRecorder() {}; @Mock NameResolver.Listener mockListener; @Test @@ -77,6 +78,7 @@ public void args() { assertThat(args.getChannelLogger()).isSameInstanceAs(channelLogger); assertThat(args.getOffloadExecutor()).isSameInstanceAs(executor); assertThat(args.getOverrideAuthority()).isSameInstanceAs(overrideAuthority); + assertThat(args.getMetricRecorder()).isSameInstanceAs(metricRecorder); NameResolver.Args args2 = args.toBuilder().build(); assertThat(args2.getDefaultPort()).isEqualTo(defaultPort); @@ -87,6 +89,7 @@ public void args() { assertThat(args2.getChannelLogger()).isSameInstanceAs(channelLogger); assertThat(args2.getOffloadExecutor()).isSameInstanceAs(executor); assertThat(args2.getOverrideAuthority()).isSameInstanceAs(overrideAuthority); + assertThat(args.getMetricRecorder()).isSameInstanceAs(metricRecorder); assertThat(args2).isNotSameInstanceAs(args); assertThat(args2).isNotEqualTo(args); @@ -102,6 +105,7 @@ private NameResolver.Args createArgs() { .setChannelLogger(channelLogger) .setOffloadExecutor(executor) .setOverrideAuthority(overrideAuthority) + .setMetricRecorder(metricRecorder) .build(); } diff --git a/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java b/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java index 16700096827..535086716bd 100644 --- a/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java +++ b/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java @@ -687,6 +687,30 @@ public void metricRecorder_recordsToMetricSink() { eq(optionalLabelValues)); } + @Test + public void metricRecorder_fromNameResolverArgs_recordsToMetricSink() { + MetricSink mockSink1 = mock(MetricSink.class); + MetricSink mockSink2 = mock(MetricSink.class); + channelBuilder.addMetricSink(mockSink1); + channelBuilder.addMetricSink(mockSink2); + createChannel(); + + LongCounterMetricInstrument counter = metricInstrumentRegistry.registerLongCounter( + "test_counter", "Time taken by metric recorder", "s", + ImmutableList.of("grpc.method"), Collections.emptyList(), false); + List requiredLabelValues = ImmutableList.of("testMethod"); + List optionalLabelValues = Collections.emptyList(); + + NameResolver.Args args = helper.getNameResolverArgs(); + assertThat(args.getMetricRecorder()).isNotNull(); + args.getMetricRecorder() + .addLongCounter(counter, 10, requiredLabelValues, optionalLabelValues); + verify(mockSink1).addLongCounter(eq(counter), eq(10L), eq(requiredLabelValues), + eq(optionalLabelValues)); + verify(mockSink2).addLongCounter(eq(counter), eq(10L), eq(requiredLabelValues), + eq(optionalLabelValues)); + } + @Test public void shutdownWithNoTransportsEverCreated() { channelBuilder.nameResolverFactory( @@ -2240,6 +2264,7 @@ public void lbHelper_getNameResolverArgs() { assertThat(args.getSynchronizationContext()) .isSameInstanceAs(helper.getSynchronizationContext()); assertThat(args.getServiceConfigParser()).isNotNull(); + assertThat(args.getMetricRecorder()).isNotNull(); } @Test diff --git a/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java b/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java index 88d015150e9..2cc4e8d917a 100644 --- a/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java +++ b/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java @@ -124,6 +124,11 @@ public ImmutableList getTargets() { return ImmutableList.copyOf(targetToXdsClientMap.keySet()); } + @VisibleForTesting + MetricRecorder getMetricRecorder(String target) { + return targetToMetricRecorderMap.get(target); + } + private static class SharedXdsClientPoolProviderHolder { private static final SharedXdsClientPoolProvider instance = new SharedXdsClientPoolProvider(); diff --git a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java index 1188e3da1d6..5a48ee86643 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java @@ -53,6 +53,8 @@ public class XdsClientMetricReporterImpl implements XdsClientMetricReporter { private Registration gaugeRegistration = null; @Nullable private XdsClient xdsClient = null; + @Nullable + private CallbackMetricReporter callbackMetricReporter = null; static { MetricInstrumentRegistry metricInstrumentRegistry @@ -129,12 +131,18 @@ public void accept(BatchRecorder recorder) { @VisibleForTesting void reportCallbackMetrics(BatchRecorder recorder) { - CallbackMetricReporter callbackMetricReporter = new CallbackMetricReporterImpl(recorder); + if (callbackMetricReporter == null) { + // Instantiate only if not injected + callbackMetricReporter = new CallbackMetricReporterImpl(recorder); + } try { reportResourceCounts(callbackMetricReporter); reportServerConnections(callbackMetricReporter); } catch (Exception e) { - logger.log(Level.WARNING, "Reporting gauge metrics failed", e); + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); // re-set the current thread's interruption state + } + logger.log(Level.WARNING, "Failed to report gauge metrics", e); } } @@ -147,7 +155,7 @@ void reportResourceCounts(CallbackMetricReporter callbackMetricReporter) throws SettableFuture ret = this.xdsClient.reportResourceCounts( callbackMetricReporter); // Normally this shouldn't take long, but adding a timeout to avoid indefinite blocking - ret.get(5, TimeUnit.SECONDS); + Void unused = ret.get(5, TimeUnit.SECONDS); } /** @@ -158,7 +166,15 @@ void reportResourceCounts(CallbackMetricReporter callbackMetricReporter) throws void reportServerConnections(CallbackMetricReporter callbackMetricReporter) throws Exception { SettableFuture ret = this.xdsClient.reportServerConnections(callbackMetricReporter); // Normally this shouldn't take long, but adding a timeout to avoid indefinite blocking - ret.get(5, TimeUnit.SECONDS); + Void unused = ret.get(5, TimeUnit.SECONDS); + } + + /** + * Allows injecting a custom {@link CallbackMetricReporter} for testing purposes. + */ + @VisibleForTesting + void injectCallbackMetricReporter(CallbackMetricReporter reporter) { + this.callbackMetricReporter = reporter; } @VisibleForTesting diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java index 7f576d48711..40487dbfc89 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java @@ -143,7 +143,7 @@ final class XdsNameResolver extends NameResolver { ServiceConfigParser serviceConfigParser, SynchronizationContext syncContext, ScheduledExecutorService scheduler, @Nullable Map bootstrapOverride, - @Nullable MetricRecorder metricRecorder) { + MetricRecorder metricRecorder) { this(targetUri, targetUri.getAuthority(), name, overrideAuthority, serviceConfigParser, syncContext, scheduler, SharedXdsClientPoolProvider.getDefaultProvider(), ThreadSafeRandomImpl.instance, FilterRegistry.getDefaultRegistry(), bootstrapOverride, @@ -157,7 +157,7 @@ final class XdsNameResolver extends NameResolver { SynchronizationContext syncContext, ScheduledExecutorService scheduler, XdsClientPoolFactory xdsClientPoolFactory, ThreadSafeRandom random, FilterRegistry filterRegistry, @Nullable Map bootstrapOverride, - @Nullable MetricRecorder metricRecorder) { + MetricRecorder metricRecorder) { this.targetAuthority = targetAuthority; target = targetUri.toString(); diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java index bd324a40f71..88eae965e4e 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java @@ -547,8 +547,8 @@ public SettableFuture reportServerConnections( controlPlaneClient.hasWorkingAdsStream() ? 1 : 0, target, serverInfo.target())); - future.set(null); }); + future.set(null); return future; } @@ -559,8 +559,8 @@ public SettableFuture reportResourceCounts(CallbackMetricReporter callback Map, Map> resourceCountsByType = getResourceCountsByType(); reportResourceCountsToCallback(callbackMetricReporter, resourceCountsByType); - future.set(null); }); + future.set(null); return future; } @@ -571,7 +571,7 @@ private String cacheStateFromResourceStatus(ResourceMetadata metadata, boolean i } /** - * Calculates resource counts by ResourceType and ResourceSubscriber.metadata.status + * Calculates number of resources by ResourceType and ResourceSubscriber.metadata.status */ Map, Map> getResourceCountsByType() { Map, Map> resourceCountsByType = new HashMap<>(); diff --git a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java index 0b41b2b1e32..994e1bd747d 100644 --- a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java +++ b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java @@ -616,7 +616,7 @@ private void validateGoldenClusterLoadAssignment(EdsUpdate edsUpdate) { * the expected number of times with the expected values for valid resource count, invalid * resource count, and corresponding metric labels. */ - private void verifyResourceValidInvalidMetrics(int times, long validResourceCount, + private void verifyResourceValidInvalidCount(int times, long validResourceCount, long invalidResourceCount, String dataPlaneTargetLabel, String xdsServerTargetLabel, String resourceType) { verify(xdsClientMetricReporter, times(times)).reportResourceUpdates( @@ -627,7 +627,19 @@ private void verifyResourceValidInvalidMetrics(int times, long validResourceCoun eq(resourceType)); } - private void reportResourceCount() { + private void verifyServerFailureCount(int times, long serverFailureCount, String dataPlaneTarget, + String xdsServer) { + verify(xdsClientMetricReporter, times(times)).reportServerFailure( + eq(serverFailureCount), + eq(dataPlaneTarget), + eq(xdsServer)); + } + + /** + * Invokes the callback, which will be called by {@link XdsClientMetricReporter} to record + * number of resources in each cacheState {@link ResourceMetadata#getStatus()}. + */ + private void callback_ReportResourceCount() { try { Future unused = xdsClient.reportResourceCounts(callbackMetricReporter); } catch (Exception e) { @@ -638,7 +650,20 @@ private void reportResourceCount() { } } - private void reportServerConnection() { + private void verifyResourceCountByCacheState(int times, long resourceCount, + String cacheState, String resourceTypeUrl, String dataPlaneTarget) { + verify(callbackMetricReporter, times(times)).reportResourceCounts( + eq(resourceCount), + eq(cacheState), + eq(resourceTypeUrl), + eq(dataPlaneTarget)); + } + + /** + * Invokes the callback, which will be called by {@link XdsClientMetricReporter} to record + * whether XdsClient has a working ADS stream. + */ + private void callback_ReportServerConnection() { try { Future unused = xdsClient.reportServerConnections(callbackMetricReporter); } catch (Exception e) { @@ -649,6 +674,14 @@ private void reportServerConnection() { } } + private void verifyServerConnection(int times, int isConnected, String dataPlaneTarget, + String xdsServer) { + verify(callbackMetricReporter, times(times)).reportServerConnections( + eq(isConnected), + eq(dataPlaneTarget), + eq(xdsServer)); + } + @Test public void ldsResourceNotFound() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), LDS_RESOURCE, @@ -662,19 +695,19 @@ public void ldsResourceNotFound() { call.verifyRequest(LDS, LDS_RESOURCE, VERSION_1, "0000", NODE); verifyNoInteractions(ldsResourceWatcher); verifyResourceMetadataRequested(LDS, LDS_RESOURCE); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), - CHANNEL_TARGET); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); // Server failed to return subscribed resource within expected time window. fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); verify(ldsResourceWatcher).onResourceDoesNotExist(LDS_RESOURCE); assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); verifyResourceMetadataDoesNotExist(LDS, LDS_RESOURCE); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "does_not_exist", LDS.typeUrl(), - CHANNEL_TARGET); + // Check metric data. verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl(), CHANNEL_TARGET); verifyNoMoreInteractions(callbackMetricReporter); } @@ -682,7 +715,7 @@ public void ldsResourceNotFound() { public void ldsResourceUpdated_withXdstpResourceName_withUnknownAuthority() { String ldsResourceName = "xdstp://unknown.example.com/envoy.config.listener.v3.Listener/listener1"; - xdsClient.watchXdsResource(XdsListenerResource.getInstance(),ldsResourceName, + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), ldsResourceName, ldsResourceWatcher); verify(ldsResourceWatcher).onError(errorCaptor.capture()); Status error = errorCaptor.getValue(); @@ -690,7 +723,7 @@ public void ldsResourceUpdated_withXdstpResourceName_withUnknownAuthority() { assertThat(error.getDescription()).isEqualTo( "Wrong configuration: xds server does not exist for resource " + ldsResourceName); assertThat(resourceDiscoveryCalls.poll()).isNull(); - xdsClient.cancelXdsResourceWatch(XdsListenerResource.getInstance(),ldsResourceName, + xdsClient.cancelXdsResourceWatch(XdsListenerResource.getInstance(), ldsResourceName, ldsResourceWatcher); assertThat(resourceDiscoveryCalls.poll()).isNull(); } @@ -743,19 +776,18 @@ public void ldsResponseErrorHandling_someResourcesFailedUnpack() { @Test public void ldsResponseErrorHandling_subscribedResourceInvalid() { List subscribedResourceNames = ImmutableList.of("A", "B", "C"); - xdsClient.watchXdsResource(XdsListenerResource.getInstance(),"A", ldsResourceWatcher); - xdsClient.watchXdsResource(XdsListenerResource.getInstance(),"B", ldsResourceWatcher); - xdsClient.watchXdsResource(XdsListenerResource.getInstance(),"C", ldsResourceWatcher); + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), "A", ldsResourceWatcher); + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), "B", ldsResourceWatcher); + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), "C", ldsResourceWatcher); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); assertThat(call).isNotNull(); verifyResourceMetadataRequested(LDS, "A"); verifyResourceMetadataRequested(LDS, "B"); verifyResourceMetadataRequested(LDS, "C"); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(3, "requested", LDS.typeUrl(), - CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(3, 0, 0, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 3, "requested", LDS.typeUrl(), CHANNEL_TARGET); // LDS -> {A, B, C}, version 1 ImmutableMap resourcesV1 = ImmutableMap.of( @@ -764,14 +796,14 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid() { "C", Any.pack(mf.buildListenerWithApiListenerForRds("C", "C.1"))); call.sendResponse(LDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 - verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), - LDS.typeUrl()); verifyResourceMetadataAcked(LDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(3, "acked", LDS.typeUrl(), CHANNEL_TARGET - ); + // Check metric data. + verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + LDS.typeUrl()); + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 3, "acked", LDS.typeUrl(), CHANNEL_TARGET); call.verifyRequest(LDS, subscribedResourceNames, VERSION_1, "0000", NODE); // LDS -> {A, B}, version 2 @@ -783,32 +815,27 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid() { // {A} -> ACK, version 2 // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> does not exist - verifyResourceValidInvalidMetrics(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), - LDS.typeUrl()); List errorsV2 = ImmutableList.of("LDS response Listener 'B' validation error: "); verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataNacked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, VERSION_2, TIME_INCREMENT * 2, errorsV2); + // Check metric data. + verifyResourceValidInvalidCount(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), + LDS.typeUrl()); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(LDS, "C"); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", LDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(1, "does_not_exist", LDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl(), CHANNEL_TARGET); } else { // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(2, "acked", LDS.typeUrl(), CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", LDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl(), CHANNEL_TARGET); } call.verifyRequestNack(LDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); @@ -819,40 +846,38 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid() { call.sendResponse(LDS, resourcesV3.values().asList(), VERSION_3, "0002"); // {A} -> does not exist // {B, C} -> ACK, version 3 - verifyResourceValidInvalidMetrics(1, 2, 0, CHANNEL_TARGET, xdsServerInfo.target(), + // Check metric data. + verifyResourceValidInvalidCount(1, 2, 0, CHANNEL_TARGET, xdsServerInfo.target(), LDS.typeUrl()); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(LDS, "A"); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "does_not_exist", - LDS.typeUrl(), CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(2, "acked", LDS.typeUrl(), CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 1, "does_not_exist", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl(), CHANNEL_TARGET); } else { // When resource deletion is disabled, {A} stays ACKed in the previous version VERSION_2. verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", LDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 3, "acked", LDS.typeUrl(), + CHANNEL_TARGET); } verifyResourceMetadataAcked(LDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); verifyResourceMetadataAcked(LDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); call.verifyRequest(LDS, subscribedResourceNames, VERSION_3, "0002", NODE); verifySubscribedResourcesMetadataSizes(3, 0, 0, 0); - verifyNoMoreInteractions(callbackMetricReporter); } @Test public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscription() { List subscribedResourceNames = ImmutableList.of("A", "B", "C"); - xdsClient.watchXdsResource(XdsListenerResource.getInstance(),"A", ldsResourceWatcher); - xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(),"A.1", rdsResourceWatcher); - xdsClient.watchXdsResource(XdsListenerResource.getInstance(),"B", ldsResourceWatcher); - xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(),"B.1", rdsResourceWatcher); - xdsClient.watchXdsResource(XdsListenerResource.getInstance(),"C", ldsResourceWatcher); - xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(),"C.1", rdsResourceWatcher); + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), "A", ldsResourceWatcher); + xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), "A.1", rdsResourceWatcher); + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), "B", ldsResourceWatcher); + xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), "B.1", rdsResourceWatcher); + xdsClient.watchXdsResource(XdsListenerResource.getInstance(), "C", ldsResourceWatcher); + xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), "C.1", rdsResourceWatcher); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); assertThat(call).isNotNull(); verifyResourceMetadataRequested(LDS, "A"); @@ -861,14 +886,11 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti verifyResourceMetadataRequested(RDS, "A.1"); verifyResourceMetadataRequested(RDS, "B.1"); verifyResourceMetadataRequested(RDS, "C.1"); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(3, "requested", LDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(3, "requested", RDS.typeUrl(), - CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(3, 0, 3, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 3, "requested", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 3, "requested", RDS.typeUrl(), CHANNEL_TARGET); // LDS -> {A, B, C}, version 1 ImmutableMap resourcesV1 = ImmutableMap.of( @@ -877,16 +899,15 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti "C", Any.pack(mf.buildListenerWithApiListenerForRds("C", "C.1"))); call.sendResponse(LDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 - verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), LDS.typeUrl()); verifyResourceMetadataAcked(LDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(3, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "requested", RDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 3, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 3, "requested", RDS.typeUrl(), CHANNEL_TARGET); call.verifyRequest(LDS, subscribedResourceNames, VERSION_1, "0000", NODE); // RDS -> {A.1, B.1, C.1}, version 1 @@ -897,15 +918,15 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti "C.1", Any.pack(mf.buildRouteConfiguration("C.1", vhostsV1))); call.sendResponse(RDS, resourcesV11.values().asList(), VERSION_1, "0000"); // {A.1, B.1, C.1} -> ACK, version 1 - verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), - RDS.typeUrl()); verifyResourceMetadataAcked(RDS, "A.1", resourcesV11.get("A.1"), VERSION_1, TIME_INCREMENT * 2); verifyResourceMetadataAcked(RDS, "B.1", resourcesV11.get("B.1"), VERSION_1, TIME_INCREMENT * 2); verifyResourceMetadataAcked(RDS, "C.1", resourcesV11.get("C.1"), VERSION_1, TIME_INCREMENT * 2); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", LDS.typeUrl(), - CHANNEL_TARGET); - verify(callbackMetricReporter).reportResourceCounts(3, "acked", RDS.typeUrl(), CHANNEL_TARGET); + // Check metric data. + verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + RDS.typeUrl()); + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 3, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 3, "acked", RDS.typeUrl(), CHANNEL_TARGET); // LDS -> {A, B}, version 2 // Failed to parse endpoint B @@ -916,7 +937,8 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti // {A} -> ACK, version 2 // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> does not exist - verifyResourceValidInvalidMetrics(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), + // Check metric data. + verifyResourceValidInvalidCount(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), LDS.typeUrl()); List errorsV2 = ImmutableList.of("LDS response Listener 'B' validation error: "); verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 3); @@ -925,25 +947,20 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti errorsV2); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(LDS, "C"); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), - CHANNEL_TARGET); - verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", LDS.typeUrl(), - CHANNEL_TARGET); - verify(callbackMetricReporter).reportResourceCounts(1, "does_not_exist", LDS.typeUrl(), - CHANNEL_TARGET); - verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", RDS.typeUrl(), - CHANNEL_TARGET); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 3, "acked", RDS.typeUrl(), CHANNEL_TARGET); } else { // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(2, "acked", LDS.typeUrl(), - CHANNEL_TARGET); - verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", LDS.typeUrl(), - CHANNEL_TARGET); - verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", RDS.typeUrl(), - CHANNEL_TARGET); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 3, "acked", RDS.typeUrl(), CHANNEL_TARGET); } call.verifyRequestNack(LDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); // {A.1} -> version 1 @@ -993,11 +1010,11 @@ public void wrappedLdsResource_preferWrappedName() { ldsResourceWatcher); Any innerResource = Any.pack(mf.buildListenerWithApiListener("random_name" /* name */, - mf.buildRouteConfiguration("do not care", mf.buildOpaqueVirtualHosts(VHOST_SIZE)))); + mf.buildRouteConfiguration("do not care", mf.buildOpaqueVirtualHosts(VHOST_SIZE)))); // Client sends an ACK LDS request. call.sendResponse(LDS, mf.buildWrappedResourceWithName(innerResource, LDS_RESOURCE), VERSION_1, - "0000"); + "0000"); call.verifyRequest(LDS, LDS_RESOURCE, VERSION_1, "0000", NODE); verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); verifyGoldenListenerVhosts(ldsUpdateCaptor.getValue()); @@ -1010,9 +1027,9 @@ public void wrappedLdsResource_preferWrappedName() { public void ldsResourceFound_containsRdsName() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), - CHANNEL_TARGET); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); call.sendResponse(LDS, testListenerRds, VERSION_1, "0000"); // Client sends an ACK LDS request. @@ -1020,10 +1037,11 @@ public void ldsResourceFound_containsRdsName() { verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); verifyGoldenListenerRds(ldsUpdateCaptor.getValue()); assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerRds, VERSION_1, TIME_INCREMENT); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); verifyNoMoreInteractions(callbackMetricReporter); } @@ -1067,10 +1085,9 @@ public void ldsResourceUpdated() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); verifyResourceMetadataRequested(LDS, LDS_RESOURCE); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); // Initial LDS response. call.sendResponse(LDS, testListenerVhosts, VERSION_1, "0000"); @@ -1078,8 +1095,9 @@ public void ldsResourceUpdated() { verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); verifyGoldenListenerVhosts(ldsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); // Updated LDS response. call.sendResponse(LDS, testListenerRds, VERSION_2, "0001"); @@ -1087,10 +1105,10 @@ public void ldsResourceUpdated() { verify(ldsResourceWatcher, times(2)).onChanged(ldsUpdateCaptor.capture()); verifyGoldenListenerRds(ldsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerRds, VERSION_2, TIME_INCREMENT * 2); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", LDS.typeUrl(), - CHANNEL_TARGET); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); assertThat(channelForCustomAuthority).isNull(); assertThat(channelForEmptyAuthority).isNull(); verifyNoMoreInteractions(callbackMetricReporter); @@ -1101,10 +1119,9 @@ public void cancelResourceWatcherNotRemoveUrlSubscribers() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); verifyResourceMetadataRequested(LDS, LDS_RESOURCE); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); // Initial LDS response. call.sendResponse(LDS, testListenerVhosts, VERSION_1, "0000"); @@ -1112,8 +1129,9 @@ public void cancelResourceWatcherNotRemoveUrlSubscribers() { verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); verifyGoldenListenerVhosts(ldsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); xdsClient.watchXdsResource(XdsListenerResource.getInstance(), LDS_RESOURCE + "1", ldsResourceWatcher); @@ -1129,10 +1147,9 @@ public void cancelResourceWatcherNotRemoveUrlSubscribers() { verifyGoldenListenerVhosts(ldsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts2, VERSION_2, TIME_INCREMENT * 2); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", LDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); verifyNoMoreInteractions(callbackMetricReporter); } @@ -1198,10 +1215,9 @@ public void ldsResourceUpdated_withXdstpResourceName_withWrongType() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), ldsResourceName, ldsResourceWatcher); assertThat(channelForCustomAuthority).isNotNull(); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); String ldsResourceNameWithWrongType = "xdstp://authority.xds.com/envoy.config.route.v3.RouteConfiguration/listener1"; @@ -1212,11 +1228,10 @@ public void ldsResourceUpdated_withXdstpResourceName_withWrongType() { call.verifyRequestNack( LDS, ldsResourceName, "", "0000", NODE, ImmutableList.of( - "Unsupported resource name: " + ldsResourceNameWithWrongType + " for type: LDS")); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "requested", LDS.typeUrl(), - CHANNEL_TARGET - ); + "Unsupported resource name: " + ldsResourceNameWithWrongType + " for type: LDS")); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); verifyNoMoreInteractions(callbackMetricReporter); } @@ -1243,7 +1258,7 @@ public void rdsResourceUpdated_withXdstpResourceName_withWrongType() { public void rdsResourceUpdated_withXdstpResourceName_unknownAuthority() { String rdsResourceName = "xdstp://unknown.example.com/envoy.config.route.v3.RouteConfiguration/route1"; - xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(),rdsResourceName, + xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), rdsResourceName, rdsResourceWatcher); verify(rdsResourceWatcher).onError(errorCaptor.capture()); Status error = errorCaptor.getValue(); @@ -1252,7 +1267,7 @@ public void rdsResourceUpdated_withXdstpResourceName_unknownAuthority() { "Wrong configuration: xds server does not exist for resource " + rdsResourceName); assertThat(resourceDiscoveryCalls.size()).isEqualTo(0); xdsClient.cancelXdsResourceWatch( - XdsRouteConfigureResource.getInstance(),rdsResourceName, rdsResourceWatcher); + XdsRouteConfigureResource.getInstance(), rdsResourceName, rdsResourceWatcher); assertThat(resourceDiscoveryCalls.size()).isEqualTo(0); } @@ -1335,10 +1350,9 @@ public void edsResourceUpdated_withXdstpResourceName_unknownAuthority() { public void ldsResourceUpdate_withFaultInjection() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); Any listener = Any.pack( mf.buildListenerWithApiListener( LDS_RESOURCE, @@ -1376,10 +1390,10 @@ public void ldsResourceUpdate_withFaultInjection() { call.verifyRequest(LDS, LDS_RESOURCE, VERSION_1, "0000", NODE); verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, listener, VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); LdsUpdate ldsUpdate = ldsUpdateCaptor.getValue(); assertThat(ldsUpdate.httpConnectionManager().virtualHosts()).hasSize(2); @@ -1411,10 +1425,9 @@ public void ldsResourceDeleted() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); verifyResourceMetadataRequested(LDS, LDS_RESOURCE); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); // Initial LDS response. call.sendResponse(LDS, testListenerVhosts, VERSION_1, "0000"); @@ -1422,21 +1435,20 @@ public void ldsResourceDeleted() { verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); verifyGoldenListenerVhosts(ldsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); // Empty LDS response deletes the listener. call.sendResponse(LDS, Collections.emptyList(), VERSION_2, "0001"); call.verifyRequest(LDS, LDS_RESOURCE, VERSION_2, "0001", NODE); verify(ldsResourceWatcher).onResourceDoesNotExist(LDS_RESOURCE); verifyResourceMetadataDoesNotExist(LDS, LDS_RESOURCE); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "does_not_exist", LDS.typeUrl(), - CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl(), CHANNEL_TARGET); verifyNoMoreInteractions(callbackMetricReporter); } @@ -1451,10 +1463,9 @@ public void ldsResourceDeleted_ignoreResourceDeletion() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); verifyResourceMetadataRequested(LDS, LDS_RESOURCE); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); // Initial LDS response. call.sendResponse(LDS, testListenerVhosts, VERSION_1, "0000"); @@ -1462,21 +1473,20 @@ public void ldsResourceDeleted_ignoreResourceDeletion() { verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); verifyGoldenListenerVhosts(ldsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); // Empty LDS response does not delete the listener. call.sendResponse(LDS, Collections.emptyList(), VERSION_2, "0001"); call.verifyRequest(LDS, LDS_RESOURCE, VERSION_2, "0001", NODE); // The resource is still ACKED at VERSION_1 (no changes). verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", LDS.typeUrl(), - CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); // onResourceDoesNotExist not called verify(ldsResourceWatcher, never()).onResourceDoesNotExist(LDS_RESOURCE); @@ -1488,11 +1498,10 @@ public void ldsResourceDeleted_ignoreResourceDeletion() { // LDS is now ACKEd at VERSION_3. verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_3, TIME_INCREMENT * 3); - reportResourceCount(); - verify(callbackMetricReporter, times(3)).reportResourceCounts(1, "acked", LDS.typeUrl(), - CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(3, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); verifyNoMoreInteractions(ldsResourceWatcher); verifyNoMoreInteractions(callbackMetricReporter); } @@ -1511,11 +1520,10 @@ public void multipleLdsWatchers() { // Both LDS resources were requested. verifyResourceMetadataRequested(LDS, LDS_RESOURCE); verifyResourceMetadataRequested(LDS, ldsResourceTwo); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(2, "requested", LDS.typeUrl(), - CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(2, 0, 0, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 2, "requested", LDS.typeUrl(), CHANNEL_TARGET); fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); verify(ldsResourceWatcher).onResourceDoesNotExist(LDS_RESOURCE); @@ -1523,11 +1531,10 @@ public void multipleLdsWatchers() { verify(watcher2).onResourceDoesNotExist(ldsResourceTwo); verifyResourceMetadataDoesNotExist(LDS, LDS_RESOURCE); verifyResourceMetadataDoesNotExist(LDS, ldsResourceTwo); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(2, "does_not_exist", LDS.typeUrl(), - CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(2, 0, 0, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 2, "does_not_exist", LDS.typeUrl(), CHANNEL_TARGET); Any listenerTwo = Any.pack(mf.buildListenerWithApiListenerForRds(ldsResourceTwo, RDS_RESOURCE)); call.sendResponse(LDS, ImmutableList.of(testListenerVhosts, listenerTwo), VERSION_1, "0000"); @@ -1545,10 +1552,10 @@ public void multipleLdsWatchers() { // Metadata of both listeners is stored. verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, ldsResourceTwo, listenerTwo, VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(2, "acked", LDS.typeUrl(), CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(2, 0, 0, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl(), CHANNEL_TARGET); verifyNoMoreInteractions(callbackMetricReporter); } @@ -1616,10 +1623,9 @@ public void rdsResponseErrorHandling_nackWeightedSumZero() { DiscoveryRpcCall call = startResourceWatcher(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, rdsResourceWatcher); verifyResourceMetadataRequested(RDS, RDS_RESOURCE); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", RDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl(), CHANNEL_TARGET); io.envoyproxy.envoy.config.route.v3.RouteAction routeAction = io.envoyproxy.envoy.config.route.v3.RouteAction.newBuilder() @@ -1654,10 +1660,10 @@ public void rdsResponseErrorHandling_nackWeightedSumZero() { + "RouteConfiguration contains invalid virtual host: Virtual host [do not care] " + "contains invalid route : Route [route-blade] contains invalid RouteAction: " + "Sum of cluster weights should be above 0."); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "nacked", RDS.typeUrl(), CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(0, 0, 1, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "nacked", RDS.typeUrl(), CHANNEL_TARGET); // The response is NACKed with the same error message. call.verifyRequestNack(RDS, RDS_RESOURCE, "", "0000", NODE, errors); verify(rdsResourceWatcher, never()).onChanged(any(RdsUpdate.class)); @@ -1682,11 +1688,10 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataRequested(RDS, "A"); verifyResourceMetadataRequested(RDS, "B"); verifyResourceMetadataRequested(RDS, "C"); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(3, "requested", RDS.typeUrl(), - CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(0, 0, 3, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 3, "requested", RDS.typeUrl(), CHANNEL_TARGET); // RDS -> {A, B, C}, version 1 List vhostsV1 = mf.buildOpaqueVirtualHosts(1); @@ -1696,13 +1701,13 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { "C", Any.pack(mf.buildRouteConfiguration("C", vhostsV1))); call.sendResponse(RDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 - verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), - RDS.typeUrl()); verifyResourceMetadataAcked(RDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(RDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(RDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(3, "acked", RDS.typeUrl(), CHANNEL_TARGET); + // Check metric data. + verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), RDS.typeUrl()); + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 3, "acked", RDS.typeUrl(), CHANNEL_TARGET); call.verifyRequest(RDS, subscribedResourceNames, VERSION_1, "0000", NODE); // RDS -> {A, B}, version 2 @@ -1714,7 +1719,7 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { // {A} -> ACK, version 2 // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> ACK, version 1 - verifyResourceValidInvalidMetrics(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), RDS.typeUrl()); List errorsV2 = ImmutableList.of("RDS response RouteConfiguration 'B' validation error: "); @@ -1722,11 +1727,10 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataNacked(RDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, VERSION_2, TIME_INCREMENT * 2, errorsV2); verifyResourceMetadataAcked(RDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(2, "acked", RDS.typeUrl(), CHANNEL_TARGET); - verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", RDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 2, "acked", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", RDS.typeUrl(), CHANNEL_TARGET); call.verifyRequestNack(RDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); // RDS -> {B, C} version 3 @@ -1737,16 +1741,15 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { call.sendResponse(RDS, resourcesV3.values().asList(), VERSION_3, "0002"); // {A} -> ACK, version 2 // {B, C} -> ACK, version 3 - verifyResourceValidInvalidMetrics(1, 2, 0, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 2, 0, CHANNEL_TARGET, xdsServerInfo.target(), RDS.typeUrl()); verifyResourceMetadataAcked(RDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataAcked(RDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); verifyResourceMetadataAcked(RDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", RDS.typeUrl(), - CHANNEL_TARGET - ); call.verifyRequest(RDS, subscribedResourceNames, VERSION_3, "0002", NODE); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 3, "acked", RDS.typeUrl(), CHANNEL_TARGET); verifySubscribedResourcesMetadataSizes(0, 0, 3, 0); verifyNoMoreInteractions(callbackMetricReporter); } @@ -1821,10 +1824,9 @@ public void rdsResourceUpdated() { DiscoveryRpcCall call = startResourceWatcher(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, rdsResourceWatcher); verifyResourceMetadataRequested(RDS, RDS_RESOURCE); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", RDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl(), CHANNEL_TARGET); // Initial RDS response. call.sendResponse(RDS, testRouteConfig, VERSION_1, "0000"); @@ -1832,8 +1834,9 @@ public void rdsResourceUpdated() { verify(rdsResourceWatcher).onChanged(rdsUpdateCaptor.capture()); verifyGoldenRouteConfig(rdsUpdateCaptor.getValue()); verifyResourceMetadataAcked(RDS, RDS_RESOURCE, testRouteConfig, VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", RDS.typeUrl(), CHANNEL_TARGET); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); // Updated RDS response. Any routeConfigUpdated = @@ -1846,11 +1849,9 @@ public void rdsResourceUpdated() { assertThat(rdsUpdateCaptor.getValue().virtualHosts).hasSize(4); verifyResourceMetadataAcked(RDS, RDS_RESOURCE, routeConfigUpdated, VERSION_2, TIME_INCREMENT * 2); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", RDS.typeUrl(), - CHANNEL_TARGET - ); - verifySubscribedResourcesMetadataSizes(0, 0, 1, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); verifyNoMoreInteractions(callbackMetricReporter); } @@ -1862,14 +1863,11 @@ public void rdsResourceDeletedByLdsApiListener() { rdsResourceWatcher); verifyResourceMetadataRequested(LDS, LDS_RESOURCE); verifyResourceMetadataRequested(RDS, RDS_RESOURCE); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", RDS.typeUrl(), - CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl(), CHANNEL_TARGET); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); call.sendResponse(LDS, testListenerRds, VERSION_1, "0000"); @@ -1877,25 +1875,22 @@ public void rdsResourceDeletedByLdsApiListener() { verifyGoldenListenerRds(ldsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerRds, VERSION_1, TIME_INCREMENT); verifyResourceMetadataRequested(RDS, RDS_RESOURCE); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "requested", RDS.typeUrl(), - CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "requested", RDS.typeUrl(), CHANNEL_TARGET); call.sendResponse(RDS, testRouteConfig, VERSION_1, "0000"); verify(rdsResourceWatcher).onChanged(rdsUpdateCaptor.capture()); verifyGoldenRouteConfig(rdsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerRds, VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(RDS, RDS_RESOURCE, testRouteConfig, VERSION_1, TIME_INCREMENT * 2); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", LDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", RDS.typeUrl(), CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); // The Listener is getting replaced configured with an RDS name, to the one configured with // vhosts. Expect the RDS resources to be discarded. @@ -1908,13 +1903,11 @@ public void rdsResourceDeletedByLdsApiListener() { verifyResourceMetadataAcked(RDS, RDS_RESOURCE, testRouteConfig, VERSION_1, TIME_INCREMENT * 2); verifyResourceMetadataAcked( LDS, LDS_RESOURCE, testListenerVhosts, VERSION_2, TIME_INCREMENT * 3); - reportResourceCount(); - verify(callbackMetricReporter, times(3)).reportResourceCounts(1, "acked", LDS.typeUrl(), - CHANNEL_TARGET); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", RDS.typeUrl(), - CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(3, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); verifyNoMoreInteractions(callbackMetricReporter); } @@ -1926,14 +1919,11 @@ public void rdsResourcesDeletedByLdsTcpListener() { rdsResourceWatcher); verifyResourceMetadataRequested(LDS, LISTENER_RESOURCE); verifyResourceMetadataRequested(RDS, RDS_RESOURCE); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", RDS.typeUrl(), - CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl(), CHANNEL_TARGET); Message hcmFilter = mf.buildHttpConnectionManagerFilter( RDS_RESOURCE, null, Collections.singletonList(mf.buildTerminalFilter())); @@ -1957,24 +1947,21 @@ public void rdsResourcesDeletedByLdsTcpListener() { assertThat(parsedFilterChain.httpConnectionManager().rdsName()).isEqualTo(RDS_RESOURCE); verifyResourceMetadataAcked(LDS, LISTENER_RESOURCE, packedListener, VERSION_1, TIME_INCREMENT); verifyResourceMetadataRequested(RDS, RDS_RESOURCE); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "requested", RDS.typeUrl(), - CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "requested", RDS.typeUrl(), CHANNEL_TARGET); // Simulates receiving the requested RDS resource. call.sendResponse(RDS, testRouteConfig, VERSION_1, "0000"); verify(rdsResourceWatcher).onChanged(rdsUpdateCaptor.capture()); verifyGoldenRouteConfig(rdsUpdateCaptor.getValue()); verifyResourceMetadataAcked(RDS, RDS_RESOURCE, testRouteConfig, VERSION_1, TIME_INCREMENT * 2); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", LDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", RDS.typeUrl(), CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); // Simulates receiving an updated version of the requested LDS resource as a TCP listener // with a filter chain containing inlined RouteConfiguration. @@ -2000,13 +1987,11 @@ public void rdsResourcesDeletedByLdsTcpListener() { verifyResourceMetadataAcked(RDS, RDS_RESOURCE, testRouteConfig, VERSION_1, TIME_INCREMENT * 2); verifyResourceMetadataAcked( LDS, LISTENER_RESOURCE, packedListener, VERSION_2, TIME_INCREMENT * 3); - reportResourceCount(); - verify(callbackMetricReporter, times(3)).reportResourceCounts(1, "acked", LDS.typeUrl(), - CHANNEL_TARGET); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", RDS.typeUrl(), - CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(3, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); verifyNoMoreInteractions(callbackMetricReporter); } @@ -2016,10 +2001,10 @@ public void multipleRdsWatchers() { String rdsResourceTwo = "route-bar.googleapis.com"; ResourceWatcher watcher1 = mock(ResourceWatcher.class); ResourceWatcher watcher2 = mock(ResourceWatcher.class); - xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(),RDS_RESOURCE, + xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, rdsResourceWatcher); - xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(),rdsResourceTwo, watcher1); - xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(),rdsResourceTwo, watcher2); + xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), rdsResourceTwo, watcher1); + xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), rdsResourceTwo, watcher2); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); call.verifyRequest(RDS, Arrays.asList(RDS_RESOURCE, rdsResourceTwo), "", "", NODE); // Both RDS resources were requested. @@ -2130,19 +2115,18 @@ public void cdsResponseErrorHandling_someResourcesFailedUnpack() { @Test public void cdsResponseErrorHandling_subscribedResourceInvalid() { List subscribedResourceNames = ImmutableList.of("A", "B", "C"); - xdsClient.watchXdsResource(XdsClusterResource.getInstance(),"A", cdsResourceWatcher); - xdsClient.watchXdsResource(XdsClusterResource.getInstance(),"B", cdsResourceWatcher); - xdsClient.watchXdsResource(XdsClusterResource.getInstance(),"C", cdsResourceWatcher); + xdsClient.watchXdsResource(XdsClusterResource.getInstance(), "A", cdsResourceWatcher); + xdsClient.watchXdsResource(XdsClusterResource.getInstance(), "B", cdsResourceWatcher); + xdsClient.watchXdsResource(XdsClusterResource.getInstance(), "C", cdsResourceWatcher); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); assertThat(call).isNotNull(); verifyResourceMetadataRequested(CDS, "A"); verifyResourceMetadataRequested(CDS, "B"); verifyResourceMetadataRequested(CDS, "C"); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(3, "requested", CDS.typeUrl(), - CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(0, 3, 0, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 3, "requested", CDS.typeUrl(), CHANNEL_TARGET); // CDS -> {A, B, C}, version 1 ImmutableMap resourcesV1 = ImmutableMap.of( @@ -2157,14 +2141,14 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { ))); call.sendResponse(CDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 - verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), CDS.typeUrl()); verifyResourceMetadataAcked(CDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(CDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(CDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(3, "acked", CDS.typeUrl(), CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 3, "acked", CDS.typeUrl(), CHANNEL_TARGET); call.verifyRequest(CDS, subscribedResourceNames, VERSION_1, "0000", NODE); // CDS -> {A, B}, version 2 @@ -2178,7 +2162,7 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { // {A} -> ACK, version 2 // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> does not exist - verifyResourceValidInvalidMetrics(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), CDS.typeUrl()); List errorsV2 = ImmutableList.of("CDS response Cluster 'B' validation error: "); verifyResourceMetadataAcked(CDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); @@ -2186,21 +2170,18 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { VERSION_2, TIME_INCREMENT * 2, errorsV2); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(CDS, "C"); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", CDS.typeUrl(), - CHANNEL_TARGET); - verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", CDS.typeUrl(), - CHANNEL_TARGET); - verify(callbackMetricReporter).reportResourceCounts(1, "does_not_exist", CDS.typeUrl(), - CHANNEL_TARGET); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "acked", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "does_not_exist", CDS.typeUrl(), CHANNEL_TARGET); } else { // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. verifyResourceMetadataAcked(CDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(2, "acked", CDS.typeUrl(), - CHANNEL_TARGET); - verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", CDS.typeUrl(), - CHANNEL_TARGET); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 2, "acked", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", CDS.typeUrl(), CHANNEL_TARGET); } call.verifyRequestNack(CDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); @@ -2215,23 +2196,20 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { call.sendResponse(CDS, resourcesV3.values().asList(), VERSION_3, "0002"); // {A} -> does not exit // {B, C} -> ACK, version 3 - verifyResourceValidInvalidMetrics(1, 2, 0, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 2, 0, CHANNEL_TARGET, xdsServerInfo.target(), CDS.typeUrl()); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(CDS, "A"); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(2, "acked", CDS.typeUrl(), CHANNEL_TARGET - ); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "does_not_exist", - CDS.typeUrl(), CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 2, "acked", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "does_not_exist", CDS.typeUrl(), CHANNEL_TARGET); } else { // When resource deletion is disabled, {A} stays ACKed in the previous version VERSION_2. verifyResourceMetadataAcked(CDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", CDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 3, "acked", CDS.typeUrl(), CHANNEL_TARGET); } verifyResourceMetadataAcked(CDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); verifyResourceMetadataAcked(CDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); @@ -2243,12 +2221,12 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { @Test public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscription() { List subscribedResourceNames = ImmutableList.of("A", "B", "C"); - xdsClient.watchXdsResource(XdsClusterResource.getInstance(),"A", cdsResourceWatcher); - xdsClient.watchXdsResource(XdsEndpointResource.getInstance(),"A.1", edsResourceWatcher); - xdsClient.watchXdsResource(XdsClusterResource.getInstance(),"B", cdsResourceWatcher); - xdsClient.watchXdsResource(XdsEndpointResource.getInstance(),"B.1", edsResourceWatcher); - xdsClient.watchXdsResource(XdsClusterResource.getInstance(),"C", cdsResourceWatcher); - xdsClient.watchXdsResource(XdsEndpointResource.getInstance(),"C.1", edsResourceWatcher); + xdsClient.watchXdsResource(XdsClusterResource.getInstance(), "A", cdsResourceWatcher); + xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), "A.1", edsResourceWatcher); + xdsClient.watchXdsResource(XdsClusterResource.getInstance(), "B", cdsResourceWatcher); + xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), "B.1", edsResourceWatcher); + xdsClient.watchXdsResource(XdsClusterResource.getInstance(), "C", cdsResourceWatcher); + xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), "C.1", edsResourceWatcher); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); assertThat(call).isNotNull(); verifyResourceMetadataRequested(CDS, "A"); @@ -2257,14 +2235,11 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti verifyResourceMetadataRequested(EDS, "A.1"); verifyResourceMetadataRequested(EDS, "B.1"); verifyResourceMetadataRequested(EDS, "C.1"); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(3, "requested", CDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(3, "requested", EDS.typeUrl(), - CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(0, 3, 0, 3); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 3, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 3, "requested", EDS.typeUrl(), CHANNEL_TARGET); // CDS -> {A, B, C}, version 1 ImmutableMap resourcesV1 = ImmutableMap.of( @@ -2279,16 +2254,15 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti ))); call.sendResponse(CDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 - verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), CDS.typeUrl()); verifyResourceMetadataAcked(CDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(CDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(CDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(3, "acked", CDS.typeUrl(), CHANNEL_TARGET); - verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "requested", EDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 3, "acked", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 3, "requested", EDS.typeUrl(), CHANNEL_TARGET); call.verifyRequest(CDS, subscribedResourceNames, VERSION_1, "0000", NODE); // EDS -> {A.1, B.1, C.1}, version 1 @@ -2300,17 +2274,15 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti "C.1", Any.pack(mf.buildClusterLoadAssignment("C.1", endpointsV1, dropOverloads))); call.sendResponse(EDS, resourcesV11.values().asList(), VERSION_1, "0000"); // {A.1, B.1, C.1} -> ACK, version 1 - verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), EDS.typeUrl()); verifyResourceMetadataAcked(EDS, "A.1", resourcesV11.get("A.1"), VERSION_1, TIME_INCREMENT * 2); verifyResourceMetadataAcked(EDS, "B.1", resourcesV11.get("B.1"), VERSION_1, TIME_INCREMENT * 2); verifyResourceMetadataAcked(EDS, "C.1", resourcesV11.get("C.1"), VERSION_1, TIME_INCREMENT * 2); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", CDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(3, "acked", EDS.typeUrl(), CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 3, "acked", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 3, "acked", EDS.typeUrl(), CHANNEL_TARGET); // CDS -> {A, B}, version 2 // Failed to parse endpoint B @@ -2323,7 +2295,8 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti // {A} -> ACK, version 2 // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> does not exist - verifyResourceValidInvalidMetrics(1, 1, 1, + // Check metric data. + verifyResourceValidInvalidCount(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), CDS.typeUrl()); List errorsV2 = ImmutableList.of("CDS response Cluster 'B' validation error: "); verifyResourceMetadataAcked(CDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 3); @@ -2332,30 +2305,20 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti errorsV2); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(CDS, "C"); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", CDS.typeUrl(), CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", CDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(1, "does_not_exist", CDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", EDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "acked", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "does_not_exist", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 3, "acked", EDS.typeUrl(), CHANNEL_TARGET); } else { // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. verifyResourceMetadataAcked(CDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(2, "acked", CDS.typeUrl(), CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", CDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", EDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 2, "acked", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 3, "acked", EDS.typeUrl(), CHANNEL_TARGET); } call.verifyRequestNack(CDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); // {A.1} -> version 1 @@ -2600,10 +2563,9 @@ public void cdsResponseWithNewUpstreamTlsContext() { public void cdsResponseErrorHandling_badUpstreamTlsContext() { DiscoveryRpcCall call = startResourceWatcher(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", CDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); // Management server sends back CDS response with UpstreamTlsContext. List clusters = ImmutableList.of(Any @@ -2613,18 +2575,16 @@ public void cdsResponseErrorHandling_badUpstreamTlsContext() { call.sendResponse(CDS, clusters, VERSION_1, "0000"); // The response NACKed with errors indicating indices of the failed resources. - String errorMsg = "CDS response Cluster 'cluster.googleapis.com' validation error: " - + "Cluster cluster.googleapis.com: malformed UpstreamTlsContext: " - + "io.grpc.xds.client.XdsResourceType$ResourceInvalidException: " - + "ca_certificate_provider_instance or system_root_certs is required in " - + "upstream-tls-context"; + String errorMsg = "CDS response Cluster 'cluster.googleapis.com' validation error: " + + "Cluster cluster.googleapis.com: malformed UpstreamTlsContext: " + + "io.grpc.xds.client.XdsResourceType$ResourceInvalidException: " + + "ca_certificate_provider_instance is required in upstream-tls-context"; call.verifyRequestNack(CDS, CDS_RESOURCE, "", "0000", NODE, ImmutableList.of(errorMsg)); verify(cdsResourceWatcher).onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, errorMsg); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "requested", CDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); verifyNoMoreInteractions(callbackMetricReporter); } @@ -2636,10 +2596,9 @@ public void cdsResponseErrorHandling_badUpstreamTlsContext() { public void cdsResponseWithOutlierDetection() { DiscoveryRpcCall call = startResourceWatcher(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", CDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); OutlierDetection outlierDetectionXds = OutlierDetection.newBuilder() .setInterval(Durations.fromNanos(100)) @@ -2698,10 +2657,10 @@ public void cdsResponseWithOutlierDetection() { assertThat(failurePercentageEjection.requestVolume()).isEqualTo(100); verifyResourceMetadataAcked(CDS, CDS_RESOURCE, clusterEds, VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", CDS.typeUrl(), CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(0, 1, 0, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "acked", CDS.typeUrl(), CHANNEL_TARGET); verifyNoMoreInteractions(callbackMetricReporter); } @@ -2714,10 +2673,9 @@ public void cdsResponseWithInvalidOutlierDetectionNacks() { DiscoveryRpcCall call = startResourceWatcher(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", CDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); OutlierDetection outlierDetectionXds = OutlierDetection.newBuilder() .setMaxEjectionPercent(UInt32Value.of(101)).build(); @@ -2823,10 +2781,9 @@ public void validateOutlierDetection_enforcingFailurePercentageTooHigh() public void cdsResponseErrorHandling_badTransportSocketName() { DiscoveryRpcCall call = startResourceWatcher(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", CDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); // Management server sends back CDS response with UpstreamTlsContext. List clusters = ImmutableList.of(Any @@ -2840,10 +2797,9 @@ public void cdsResponseErrorHandling_badTransportSocketName() { String errorMsg = "CDS response Cluster 'cluster.googleapis.com' validation error: " + "transport-socket with name envoy.transport_sockets.bad not supported."; call.verifyRequestNack(CDS, CDS_RESOURCE, "", "0000", NODE, ImmutableList.of(errorMsg)); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "requested", CDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); verify(cdsResourceWatcher).onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, errorMsg); } @@ -2901,10 +2857,9 @@ public void cachedCdsResource_data() { public void cachedCdsResource_absent() { DiscoveryRpcCall call = startResourceWatcher(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", CDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); verify(cdsResourceWatcher).onResourceDoesNotExist(CDS_RESOURCE); ResourceWatcher watcher = mock(ResourceWatcher.class); @@ -2912,11 +2867,10 @@ public void cachedCdsResource_absent() { verify(watcher).onResourceDoesNotExist(CDS_RESOURCE); call.verifyNoMoreRequest(); verifyResourceMetadataDoesNotExist(CDS, CDS_RESOURCE); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "does_not_exist", CDS.typeUrl(), - CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(0, 1, 0, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "does_not_exist", CDS.typeUrl(), CHANNEL_TARGET); } @Test @@ -3198,10 +3152,9 @@ public void edsCleanupNonceAfterUnsubscription() { DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); assertThat(call).isNotNull(); call.verifyRequest(EDS, "A.1", "", "", NODE); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", EDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "requested", EDS.typeUrl(), CHANNEL_TARGET); // EDS -> {A.1}, version 1 List dropOverloads = ImmutableList.of(); @@ -3212,9 +3165,9 @@ public void edsCleanupNonceAfterUnsubscription() { // {A.1} -> ACK, version 1 call.verifyRequest(EDS, "A.1", VERSION_1, "0000", NODE); verify(edsResourceWatcher, times(1)).onChanged(any()); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", EDS.typeUrl(), CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "acked", EDS.typeUrl(), CHANNEL_TARGET); // trigger an EDS resource unsubscription. xdsClient.cancelXdsResourceWatch(XdsEndpointResource.getInstance(), "A.1", edsResourceWatcher); @@ -3225,10 +3178,9 @@ public void edsCleanupNonceAfterUnsubscription() { // same as the initial request xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), "A.1", edsResourceWatcher); call.verifyRequest(EDS, "A.1", "", "", NODE, Mockito.timeout(2000).times(2)); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "requested", EDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 1, "requested", EDS.typeUrl(), CHANNEL_TARGET); verifyNoMoreInteractions(callbackMetricReporter); } @@ -3291,11 +3243,9 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataRequested(EDS, "A"); verifyResourceMetadataRequested(EDS, "B"); verifyResourceMetadataRequested(EDS, "C"); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(3, "requested", EDS.typeUrl(), - CHANNEL_TARGET - ); - verifySubscribedResourcesMetadataSizes(0, 0, 0, 3); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 3, "requested", EDS.typeUrl(), CHANNEL_TARGET); // EDS -> {A, B, C}, version 1 List dropOverloads = ImmutableList.of(mf.buildDropOverload("lb", 200)); @@ -3306,13 +3256,14 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { "C", Any.pack(mf.buildClusterLoadAssignment("C", endpointsV1, dropOverloads))); call.sendResponse(EDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 - verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), EDS.typeUrl()); verifyResourceMetadataAcked(EDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(EDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(EDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(3, "acked", EDS.typeUrl(), CHANNEL_TARGET); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 3, "acked", EDS.typeUrl(), CHANNEL_TARGET); call.verifyRequest(EDS, subscribedResourceNames, VERSION_1, "0000", NODE); // EDS -> {A, B}, version 2 @@ -3325,7 +3276,8 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { // {A} -> ACK, version 2 // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> ACK, version 1 - verifyResourceValidInvalidMetrics(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), + // Check metric data. + verifyResourceValidInvalidCount(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), EDS.typeUrl()); List errorsV2 = ImmutableList.of("EDS response ClusterLoadAssignment 'B' validation error: "); @@ -3333,11 +3285,10 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataNacked(EDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, VERSION_2, TIME_INCREMENT * 2, errorsV2); verifyResourceMetadataAcked(EDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(2, "acked", EDS.typeUrl(), CHANNEL_TARGET); - verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", EDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 2, "acked", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", EDS.typeUrl(), CHANNEL_TARGET); call.verifyRequestNack(EDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); // EDS -> {B, C} version 3 @@ -3349,16 +3300,17 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { call.sendResponse(EDS, resourcesV3.values().asList(), VERSION_3, "0002"); // {A} -> ACK, version 2 // {B, C} -> ACK, version 3 - verifyResourceValidInvalidMetrics(1, 2, 0, CHANNEL_TARGET, xdsServerInfo.target(), + // Check metric data. + verifyResourceValidInvalidCount(1, 2, 0, CHANNEL_TARGET, xdsServerInfo.target(), EDS.typeUrl()); verifyResourceMetadataAcked(EDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataAcked(EDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); verifyResourceMetadataAcked(EDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", EDS.typeUrl(), - CHANNEL_TARGET); call.verifyRequest(EDS, subscribedResourceNames, VERSION_3, "0002", NODE); verifySubscribedResourcesMetadataSizes(0, 0, 0, 3); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 3, "acked", EDS.typeUrl(), CHANNEL_TARGET); verifyNoMoreInteractions(callbackMetricReporter); } @@ -3439,10 +3391,9 @@ public void flowControlAbsent() throws Exception { anotherWatcher, fakeWatchClock.getScheduledExecutorService()); verifyResourceMetadataRequested(CDS, CDS_RESOURCE); verifyResourceMetadataRequested(CDS, anotherCdsResource); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(2, "requested", CDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 2, "requested", CDS.typeUrl(), CHANNEL_TARGET); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); call.verifyRequest(CDS, Arrays.asList(CDS_RESOURCE, anotherCdsResource), "", "", NODE); @@ -3450,11 +3401,10 @@ public void flowControlAbsent() throws Exception { call.sendResponse(CDS, testClusterRoundRobin, VERSION_1, "0000"); verifyResourceMetadataAcked( CDS, CDS_RESOURCE, testClusterRoundRobin, VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", CDS.typeUrl(), CHANNEL_TARGET); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", CDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "acked", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); call.verifyRequest(CDS, Arrays.asList(CDS_RESOURCE, anotherCdsResource), VERSION_1, "0000", NODE); verifyNoInteractions(cdsResourceWatcher, anotherWatcher); @@ -3482,12 +3432,10 @@ public void flowControlAbsent() throws Exception { assertThat(call.isReady()).isFalse(); verifyResourceMetadataAcked( CDS, CDS_RESOURCE, testClusterRoundRobin, VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", CDS.typeUrl(), - CHANNEL_TARGET); - verify(callbackMetricReporter).reportResourceCounts(1, "does_not_exist", CDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 1, "acked", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "does_not_exist", CDS.typeUrl(), CHANNEL_TARGET); barrier.await(); verify(cdsResourceWatcher, atLeastOnce()).onChanged(any()); String errorMsg = "CDS response Cluster 'cluster.googleapis.com2' validation error: " @@ -3499,11 +3447,10 @@ public void flowControlAbsent() throws Exception { verify(cdsResourceWatcher, times(2)).onChanged(any()); verify(anotherWatcher).onResourceDoesNotExist(eq(anotherCdsResource)); verify(anotherWatcher).onError(any()); - reportResourceCount(); - verify(callbackMetricReporter, times(3)).reportResourceCounts(1, "acked", CDS.typeUrl(), - CHANNEL_TARGET); - verify(callbackMetricReporter).reportResourceCounts(1, "nacked", CDS.typeUrl(), CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(3, 1, "acked", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "nacked", CDS.typeUrl(), CHANNEL_TARGET); verifyNoMoreInteractions(callbackMetricReporter); } @@ -3653,14 +3600,11 @@ public void edsResourceDeletedByCds() { verifyResourceMetadataRequested(CDS, resource); verifyResourceMetadataRequested(EDS, EDS_RESOURCE); verifyResourceMetadataRequested(EDS, resource); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(2, "requested", CDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(2, "requested", EDS.typeUrl(), - CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(0, 2, 0, 2); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 2, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 2, "requested", EDS.typeUrl(), CHANNEL_TARGET); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); List clusters = ImmutableList.of( @@ -3682,12 +3626,10 @@ public void edsResourceDeletedByCds() { verifyResourceMetadataAcked(CDS, CDS_RESOURCE, clusters.get(1), VERSION_1, TIME_INCREMENT); verifyResourceMetadataRequested(EDS, EDS_RESOURCE); verifyResourceMetadataRequested(EDS, resource); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(2, "acked", CDS.typeUrl(), CHANNEL_TARGET - ); - verify(callbackMetricReporter, times(2)).reportResourceCounts(2, "requested", EDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 2, "acked", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 2, "requested", EDS.typeUrl(), CHANNEL_TARGET); List clusterLoadAssignments = ImmutableList.of( @@ -3717,13 +3659,11 @@ public void edsResourceDeletedByCds() { // CDS not changed. verifyResourceMetadataAcked(CDS, resource, clusters.get(0), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(CDS, CDS_RESOURCE, clusters.get(1), VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(2, "acked", CDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(2, "acked", EDS.typeUrl(), CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(0, 2, 0, 2); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 2, "acked", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 2, "acked", EDS.typeUrl(), CHANNEL_TARGET); clusters = ImmutableList.of( Any.pack(mf.buildEdsCluster(resource, null, "round_robin", null, null, true, null, @@ -3745,14 +3685,10 @@ public void edsResourceDeletedByCds() { EDS, resource, clusterLoadAssignments.get(1), VERSION_1, TIME_INCREMENT * 2); // no change verifyResourceMetadataAcked(CDS, resource, clusters.get(0), VERSION_2, TIME_INCREMENT * 3); verifyResourceMetadataAcked(CDS, CDS_RESOURCE, clusters.get(1), VERSION_2, TIME_INCREMENT * 3); - reportResourceCount(); - verify(callbackMetricReporter, times(3)).reportResourceCounts(2, "acked", CDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter, times(2)).reportResourceCounts(2, "acked", EDS.typeUrl(), - CHANNEL_TARGET - ); - verifySubscribedResourcesMetadataSizes(0, 2, 0, 2); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(3, 2, "acked", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 2, "acked", EDS.typeUrl(), CHANNEL_TARGET); verifyNoMoreInteractions(callbackMetricReporter); } @@ -3853,14 +3789,14 @@ public void streamClosedWithNoResponse() { xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, rdsResourceWatcher); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); - reportServerConnection(); - verify(callbackMetricReporter).reportServerConnections(1, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(1, 1, CHANNEL_TARGET, xdsServerInfo.target()); // Management server closes the RPC stream before sending any response. call.sendCompleted(); - reportServerConnection(); - verify(callbackMetricReporter).reportServerConnections(0, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(1, 0, CHANNEL_TARGET, xdsServerInfo.target()); verify(ldsResourceWatcher, Mockito.timeout(1000).times(1)) .onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, @@ -3877,25 +3813,25 @@ public void streamClosedAfterSendingResponses() { xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, rdsResourceWatcher); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); - reportServerConnection(); - verify(callbackMetricReporter).reportServerConnections(1, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(1, 1, CHANNEL_TARGET, xdsServerInfo.target()); ScheduledTask ldsResourceTimeout = Iterables.getOnlyElement(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); ScheduledTask rdsResourceTimeout = Iterables.getOnlyElement(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); call.sendResponse(LDS, testListenerRds, VERSION_1, "0000"); - reportServerConnection(); - verify(callbackMetricReporter, times(2)).reportServerConnections(1, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(2, 1, CHANNEL_TARGET, xdsServerInfo.target()); assertThat(ldsResourceTimeout.isCancelled()).isTrue(); call.sendResponse(RDS, testRouteConfig, VERSION_1, "0000"); assertThat(rdsResourceTimeout.isCancelled()).isTrue(); // Management server closes the RPC stream after sending responses. call.sendCompleted(); - reportServerConnection(); - verify(callbackMetricReporter, times(3)).reportServerConnections(1, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(3, 1, CHANNEL_TARGET, xdsServerInfo.target()); verify(ldsResourceWatcher, never()).onError(errorCaptor.capture()); verify(rdsResourceWatcher, never()).onError(errorCaptor.capture()); } @@ -3904,9 +3840,9 @@ public void streamClosedAfterSendingResponses() { public void streamClosedAndRetryWithBackoff() { InOrder inOrder = Mockito.inOrder(backoffPolicyProvider, backoffPolicy1, backoffPolicy2); xdsClient.watchXdsResource(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); - reportServerConnection(); - verify(callbackMetricReporter).reportServerConnections(1, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(1, 1, CHANNEL_TARGET, xdsServerInfo.target()); xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, rdsResourceWatcher); xdsClient.watchXdsResource(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); @@ -3929,9 +3865,9 @@ public void streamClosedAndRetryWithBackoff() { verify(edsResourceWatcher).onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNKNOWN, ""); - reportServerConnection(); - verify(callbackMetricReporter).reportServerConnections(0, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(1, 0, CHANNEL_TARGET, xdsServerInfo.target()); // Retry after backoff. inOrder.verify(backoffPolicyProvider).get(); @@ -3946,9 +3882,9 @@ public void streamClosedAndRetryWithBackoff() { call.verifyRequest(CDS, CDS_RESOURCE, "", "", NODE); call.verifyRequest(EDS, EDS_RESOURCE, "", "", NODE); - reportServerConnection(); - verify(callbackMetricReporter, times(2)).reportServerConnections(0, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(2, 0, CHANNEL_TARGET, xdsServerInfo.target()); // Management server becomes unreachable. String errorMsg = "my fault"; @@ -3962,9 +3898,9 @@ public void streamClosedAndRetryWithBackoff() { verify(edsResourceWatcher, times(2)).onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, errorMsg); - reportServerConnection(); - verify(callbackMetricReporter, times(3)).reportServerConnections(0, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(3, 0, CHANNEL_TARGET, xdsServerInfo.target()); // Retry after backoff. inOrder.verify(backoffPolicy1).nextBackoffNanos(); @@ -3983,9 +3919,9 @@ public void streamClosedAndRetryWithBackoff() { mf.buildRouteConfiguration("do not care", mf.buildOpaqueVirtualHosts(2))))); call.sendResponse(LDS, listeners, "63", "3242"); call.verifyRequest(LDS, LDS_RESOURCE, "63", "3242", NODE); - reportServerConnection(); - verify(callbackMetricReporter, times(2)).reportServerConnections(1, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(2, 1, CHANNEL_TARGET, xdsServerInfo.target()); List routeConfigs = ImmutableList.of( Any.pack(mf.buildRouteConfiguration(RDS_RESOURCE, mf.buildOpaqueVirtualHosts(2)))); @@ -4000,9 +3936,9 @@ public void streamClosedAndRetryWithBackoff() { verify(edsResourceWatcher, times(2)).onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, errorMsg); - reportServerConnection(); - verify(callbackMetricReporter, times(3)).reportServerConnections(1, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(3, 1, CHANNEL_TARGET, xdsServerInfo.target()); // Reset backoff sequence and retry after backoff. inOrder.verify(backoffPolicyProvider).get(); @@ -4026,9 +3962,9 @@ public void streamClosedAndRetryWithBackoff() { verify(edsResourceWatcher, times(3)).onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, ""); - reportServerConnection(); - verify(callbackMetricReporter, times(4)).reportServerConnections(0, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(4, 0, CHANNEL_TARGET, xdsServerInfo.target()); // Retry after backoff. inOrder.verify(backoffPolicy2).nextBackoffNanos(); @@ -4042,9 +3978,9 @@ public void streamClosedAndRetryWithBackoff() { call.verifyRequest(CDS, CDS_RESOURCE, "", "", NODE); call.verifyRequest(EDS, EDS_RESOURCE, "", "", NODE); - reportServerConnection(); - verify(callbackMetricReporter, times(5)).reportServerConnections(0, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(5, 0, CHANNEL_TARGET, xdsServerInfo.target()); inOrder.verifyNoMoreInteractions(); verifyNoMoreInteractions(callbackMetricReporter); @@ -4057,9 +3993,9 @@ public void streamClosedAndRetryRaceWithAddRemoveWatchers() { xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, rdsResourceWatcher); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); - reportServerConnection(); - verify(callbackMetricReporter).reportServerConnections(1, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(1, 1, CHANNEL_TARGET, xdsServerInfo.target()); call.sendError(Status.UNAVAILABLE.asException()); verify(ldsResourceWatcher, Mockito.timeout(1000).times(1)) .onError(errorCaptor.capture()); @@ -4070,9 +4006,9 @@ public void streamClosedAndRetryRaceWithAddRemoveWatchers() { Iterables.getOnlyElement(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER)); assertThat(retryTask.getDelay(TimeUnit.NANOSECONDS)).isEqualTo(10L); - reportServerConnection(); - verify(callbackMetricReporter).reportServerConnections(0, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(1, 0, CHANNEL_TARGET, xdsServerInfo.target()); xdsClient.cancelXdsResourceWatch(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); @@ -4088,21 +4024,20 @@ public void streamClosedAndRetryRaceWithAddRemoveWatchers() { call.verifyRequest(EDS, EDS_RESOURCE, "", "", NODE); call.verifyNoMoreRequest(); - reportServerConnection(); - verify(callbackMetricReporter, times(2)).reportServerConnections(0, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(2,0, CHANNEL_TARGET, xdsServerInfo.target()); call.sendResponse(LDS, testListenerRds, VERSION_1, "0000"); List routeConfigs = ImmutableList.of( Any.pack(mf.buildRouteConfiguration(RDS_RESOURCE, mf.buildOpaqueVirtualHosts(VHOST_SIZE)))); call.sendResponse(RDS, routeConfigs, VERSION_1, "0000"); - reportServerConnection(); - verify(callbackMetricReporter, times(2)).reportServerConnections(1, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(2, 1, CHANNEL_TARGET, xdsServerInfo.target()); - verifyNoMoreInteractions(ldsResourceWatcher, rdsResourceWatcher); - verifyNoMoreInteractions(callbackMetricReporter); + verifyNoMoreInteractions(ldsResourceWatcher, rdsResourceWatcher, callbackMetricReporter); } @Test @@ -4113,18 +4048,14 @@ public void streamClosedAndRetryRestartsResourceInitialFetchTimerForUnresolvedRe xdsClient.watchXdsResource(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), EDS_RESOURCE, edsResourceWatcher); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); - reportResourceCount(); - reportServerConnection(); - verify(callbackMetricReporter).reportServerConnections(1, CHANNEL_TARGET, - xdsServerInfo.target()); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), - CHANNEL_TARGET); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", RDS.typeUrl(), - CHANNEL_TARGET); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", CDS.typeUrl(), - CHANNEL_TARGET); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", EDS.typeUrl(), - CHANNEL_TARGET); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(1, 1, CHANNEL_TARGET, xdsServerInfo.target()); + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", EDS.typeUrl(), CHANNEL_TARGET); ScheduledTask ldsResourceTimeout = Iterables.getOnlyElement(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); ScheduledTask rdsResourceTimeout = @@ -4135,34 +4066,25 @@ public void streamClosedAndRetryRestartsResourceInitialFetchTimerForUnresolvedRe Iterables.getOnlyElement(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); call.sendResponse(LDS, testListenerRds, VERSION_1, "0000"); assertThat(ldsResourceTimeout.isCancelled()).isTrue(); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET - ); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "requested", RDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "requested", CDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "requested", EDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(2, 1, CHANNEL_TARGET, xdsServerInfo.target()); + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "requested", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "requested", EDS.typeUrl(), CHANNEL_TARGET); call.sendResponse(RDS, testRouteConfig, VERSION_1, "0000"); assertThat(rdsResourceTimeout.isCancelled()).isTrue(); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", LDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", RDS.typeUrl(), CHANNEL_TARGET - ); - verify(callbackMetricReporter, times(3)).reportResourceCounts(1, "requested", CDS.typeUrl(), - CHANNEL_TARGET); - verify(callbackMetricReporter, times(3)).reportResourceCounts(1, "requested", EDS.typeUrl(), - CHANNEL_TARGET); - reportServerConnection(); - verify(callbackMetricReporter, times(2)).reportServerConnections(1, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(3, 1, CHANNEL_TARGET, xdsServerInfo.target()); + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(3, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(3, 1, "requested", EDS.typeUrl(), CHANNEL_TARGET); call.sendError(Status.UNAVAILABLE.asException()); assertThat(cdsResourceTimeout.isCancelled()).isTrue(); @@ -4171,22 +4093,14 @@ public void streamClosedAndRetryRestartsResourceInitialFetchTimerForUnresolvedRe verify(rdsResourceWatcher, never()).onError(errorCaptor.capture()); verify(cdsResourceWatcher, never()).onError(errorCaptor.capture()); verify(edsResourceWatcher, never()).onError(errorCaptor.capture()); - reportResourceCount(); - verify(callbackMetricReporter, times(3)).reportResourceCounts(1, "acked", LDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "acked", RDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter, times(4)).reportResourceCounts(1, "requested", CDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter, times(4)).reportResourceCounts(1, "requested", EDS.typeUrl(), - CHANNEL_TARGET - ); - reportServerConnection(); - verify(callbackMetricReporter, times(3)).reportServerConnections(1, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(4, 1, CHANNEL_TARGET, xdsServerInfo.target()); + callback_ReportResourceCount(); + verifyResourceCountByCacheState(3, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(4, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(4, 1, "requested", EDS.typeUrl(), CHANNEL_TARGET); fakeClock.forwardNanos(10L); assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(0); @@ -4349,16 +4263,12 @@ public void sendingToStoppedServer() throws Exception { xdsClient.watchXdsResource(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); fakeClock.forwardTime(14, TimeUnit.SECONDS); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "unknown", LDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", CDS.typeUrl(), - CHANNEL_TARGET - ); - reportServerConnection(); - verify(callbackMetricReporter).reportServerConnections(0, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(1, 0, CHANNEL_TARGET, xdsServerInfo.target()); + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "unknown", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); // Restart the server xdsServer = cleanupRule.register( @@ -4373,28 +4283,23 @@ public void sendingToStoppedServer() throws Exception { verify(ldsResourceWatcher, never()).onResourceDoesNotExist(LDS_RESOURCE); fakeClock.forwardTime(20, TimeUnit.SECONDS); // Trigger rpcRetryTimer DiscoveryRpcCall call = resourceDiscoveryCalls.poll(3, TimeUnit.SECONDS); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "unknown", LDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "requested", CDS.typeUrl(), - CHANNEL_TARGET - ); - reportServerConnection(); - verify(callbackMetricReporter, times(2)).reportServerConnections(0, CHANNEL_TARGET, - xdsServerInfo.target()); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(2, 0, CHANNEL_TARGET, xdsServerInfo.target()); + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 1, "unknown", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); if (call == null) { // The first rpcRetry may have happened before the channel was ready fakeClock.forwardTime(50, TimeUnit.SECONDS); call = resourceDiscoveryCalls.poll(3, TimeUnit.SECONDS); } - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "requested", LDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter, times(3)).reportResourceCounts(1, "requested", CDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(3, 0, CHANNEL_TARGET, xdsServerInfo.target()); + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(3, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); // NOTE: There is a ScheduledExecutorService that may get involved due to the reconnect // so you cannot rely on the logic being single threaded. The timeout() in verifyRequest @@ -4406,16 +4311,14 @@ public void sendingToStoppedServer() throws Exception { verifyGoldenListenerVhosts(ldsUpdateCaptor.getValue()); assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET - ); - verify(callbackMetricReporter, times(4)).reportResourceCounts(1, "requested", CDS.typeUrl(), - CHANNEL_TARGET - ); - reportServerConnection(); - verify(callbackMetricReporter).reportServerConnections(1, CHANNEL_TARGET, - xdsServerInfo.target()); verifySubscribedResourcesMetadataSizes(1, 1, 0, 0); + // Check metric data. + callback_ReportServerConnection(); + verifyServerConnection(1, 1, CHANNEL_TARGET, xdsServerInfo.target()); + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(4, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyNoMoreInteractions(callbackMetricReporter); } catch (Throwable t) { throw t; // This allows putting a breakpoint here for debugging @@ -4472,7 +4375,7 @@ public void validAndInvalidResourceMetricReport() { ))); call.sendResponse(CDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 - verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), CDS.typeUrl()); // EDS -> {A.1, B.1, C.1}, version 1 @@ -4484,7 +4387,7 @@ public void validAndInvalidResourceMetricReport() { "C.1", Any.pack(mf.buildClusterLoadAssignment("C.1", endpointsV1, dropOverloads))); call.sendResponse(EDS, resourcesV11.values().asList(), VERSION_1, "0000"); // {A.1, B.1, C.1} -> ACK, version 1 - verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), EDS.typeUrl()); // CDS -> {A, B}, version 2 @@ -4498,7 +4401,7 @@ public void validAndInvalidResourceMetricReport() { // {A} -> ACK, version 2 // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> does not exist - verifyResourceValidInvalidMetrics(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), CDS.typeUrl()); } @@ -4517,8 +4420,7 @@ public void serverFailureMetricReport() { verify(rdsResourceWatcher).onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, "ADS stream closed with OK before receiving a response"); - verify(xdsClientMetricReporter, times(1)).reportServerFailure(eq(1L), eq(CHANNEL_TARGET), eq( - xdsServerInfo.target())); + verifyServerFailureCount(1, 1, CHANNEL_TARGET, xdsServerInfo.target()); } @Test @@ -4533,8 +4435,7 @@ public void serverFailureMetricReport_forRetryAndBackoff() { // Management server closes the RPC stream with an error. call.sendError(Status.UNKNOWN.asException()); - verify(xdsClientMetricReporter, times(1)).reportServerFailure(eq(1L), eq(CHANNEL_TARGET), - eq(xdsServerInfo.target())); + verifyServerFailureCount(1, 1, CHANNEL_TARGET, xdsServerInfo.target()); // Retry after backoff. inOrder.verify(backoffPolicyProvider).get(); @@ -4548,8 +4449,7 @@ public void serverFailureMetricReport_forRetryAndBackoff() { // Management server becomes unreachable. String errorMsg = "my fault"; call.sendError(Status.UNAVAILABLE.withDescription(errorMsg).asException()); - verify(xdsClientMetricReporter, times(2)).reportServerFailure(eq(1L), eq(CHANNEL_TARGET), - eq(xdsServerInfo.target())); + verifyServerFailureCount(2, 1, CHANNEL_TARGET, xdsServerInfo.target()); // Retry after backoff. inOrder.verify(backoffPolicy1).nextBackoffNanos(); @@ -4568,8 +4468,7 @@ public void serverFailureMetricReport_forRetryAndBackoff() { call.sendError(Status.DEADLINE_EXCEEDED.asException()); // Server Failure metric will not be reported, as stream is closed with an error after receiving // a response - verify(xdsClientMetricReporter, times(2)).reportServerFailure(eq(1L), eq(CHANNEL_TARGET), - eq(xdsServerInfo.target())); + verifyServerFailureCount(2, 1, CHANNEL_TARGET, xdsServerInfo.target()); // Reset backoff sequence and retry after backoff. inOrder.verify(backoffPolicyProvider).get(); @@ -4582,8 +4481,7 @@ public void serverFailureMetricReport_forRetryAndBackoff() { // Management server becomes unreachable again. call.sendError(Status.UNAVAILABLE.asException()); - verify(xdsClientMetricReporter, times(3)).reportServerFailure(eq(1L), eq(CHANNEL_TARGET), - eq(xdsServerInfo.target())); + verifyServerFailureCount(3, 1, CHANNEL_TARGET, xdsServerInfo.target()); // Retry after backoff. inOrder.verify(backoffPolicy2).nextBackoffNanos(); @@ -4598,8 +4496,7 @@ public void serverFailureMetricReport_forRetryAndBackoff() { call.sendCompleted(); // Server Failure metric will not be reported once again, as stream is closed after receiving a // response - verify(xdsClientMetricReporter, times(3)).reportServerFailure(eq(1L), eq(CHANNEL_TARGET), - eq(xdsServerInfo.target())); + verifyServerFailureCount(3, 1, CHANNEL_TARGET, xdsServerInfo.target()); } @@ -4614,11 +4511,10 @@ public void testReportResourceCounts() { verifyResourceMetadataRequested(LDS, "A"); verifyResourceMetadataRequested(LDS, "B"); verifyResourceMetadataRequested(LDS, "C"); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(3, "requested", LDS.typeUrl(), - CHANNEL_TARGET - ); verifySubscribedResourcesMetadataSizes(3, 0, 0, 0); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 3, "requested", LDS.typeUrl(), CHANNEL_TARGET); // LDS -> {A, B, C}, version 1 ImmutableMap resourcesV1 = ImmutableMap.of( @@ -4627,14 +4523,14 @@ public void testReportResourceCounts() { "C", Any.pack(mf.buildListenerWithApiListenerForRds("C", "C.1"))); call.sendResponse(LDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 - verifyResourceValidInvalidMetrics(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), LDS.typeUrl()); verifyResourceMetadataAcked(LDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(3, "acked", LDS.typeUrl(), CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 3, "acked", LDS.typeUrl(), CHANNEL_TARGET); call.verifyRequest(LDS, subscribedResourceNames, VERSION_1, "0000", NODE); // LDS -> {A, B}, version 2 @@ -4646,7 +4542,7 @@ public void testReportResourceCounts() { // {A} -> ACK, version 2 // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> does not exist - verifyResourceValidInvalidMetrics(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), LDS.typeUrl()); List errorsV2 = ImmutableList.of("LDS response Listener 'B' validation error: "); verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); @@ -4654,24 +4550,18 @@ public void testReportResourceCounts() { VERSION_2, TIME_INCREMENT * 2, errorsV2); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(LDS, "C"); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(1, "acked", LDS.typeUrl(), CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", LDS.typeUrl(), - CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(1, "does_not_exist", LDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl(), CHANNEL_TARGET); } else { // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(2, "acked", LDS.typeUrl(), CHANNEL_TARGET - ); - verify(callbackMetricReporter).reportResourceCounts(1, "nacked_but_cached", LDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl(), CHANNEL_TARGET); } call.verifyRequestNack(LDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); @@ -4682,23 +4572,20 @@ public void testReportResourceCounts() { call.sendResponse(LDS, resourcesV3.values().asList(), VERSION_3, "0002"); // {A} -> does not exist // {B, C} -> ACK, version 3 - verifyResourceValidInvalidMetrics(1, 2, 0, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 2, 0, CHANNEL_TARGET, xdsServerInfo.target(), LDS.typeUrl()); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(LDS, "A"); - reportResourceCount(); - verify(callbackMetricReporter).reportResourceCounts(2, "acked", LDS.typeUrl(), CHANNEL_TARGET - ); - verify(callbackMetricReporter, times(2)).reportResourceCounts(1, "does_not_exist", - LDS.typeUrl(), CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "does_not_exist", LDS.typeUrl(), CHANNEL_TARGET); } else { // When resource deletion is disabled, {A} stays ACKed in the previous version VERSION_2. verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); - reportResourceCount(); - verify(callbackMetricReporter, times(2)).reportResourceCounts(3, "acked", LDS.typeUrl(), - CHANNEL_TARGET - ); + // Check metric data. + callback_ReportResourceCount(); + verifyResourceCountByCacheState(2, 3, "acked", LDS.typeUrl(), CHANNEL_TARGET); } verifyResourceMetadataAcked(LDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); verifyResourceMetadataAcked(LDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); diff --git a/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java b/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java index 2e292f479e1..b1d3db2698e 100644 --- a/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java @@ -52,6 +52,7 @@ public class SharedXdsClientPoolProviderTest { @Rule public final ExpectedException thrown = ExpectedException.none(); private final Node node = Node.newBuilder().setId("SharedXdsClientPoolProviderTest").build(); + private final MetricRecorder metricRecorder = new MetricRecorder() {}; private static final String DUMMY_TARGET = "dummy"; @Mock @@ -67,6 +68,7 @@ public void noServer() throws XdsInitializationException { thrown.expectMessage("No xDS server provided"); provider.getOrCreate(DUMMY_TARGET); assertThat(provider.get(DUMMY_TARGET)).isNull(); + assertThat(provider.getMetricRecorder(DUMMY_TARGET)).isNull(); } @Test @@ -78,11 +80,15 @@ public void sharedXdsClientObjectPool() throws XdsInitializationException { SharedXdsClientPoolProvider provider = new SharedXdsClientPoolProvider(bootstrapper); assertThat(provider.get(DUMMY_TARGET)).isNull(); + assertThat(provider.getMetricRecorder(DUMMY_TARGET)).isNull(); ObjectPool xdsClientPool = provider.getOrCreate(DUMMY_TARGET); + MetricRecorder metricRecorder = provider.getMetricRecorder(DUMMY_TARGET); verify(bootstrapper).bootstrap(); assertThat(provider.getOrCreate(DUMMY_TARGET)).isSameInstanceAs(xdsClientPool); assertThat(provider.get(DUMMY_TARGET)).isNotNull(); assertThat(provider.get(DUMMY_TARGET)).isSameInstanceAs(xdsClientPool); + assertThat(provider.getMetricRecorder(DUMMY_TARGET)).isNotNull(); + assertThat(provider.getMetricRecorder(DUMMY_TARGET)).isSameInstanceAs(metricRecorder); verifyNoMoreInteractions(bootstrapper); } @@ -92,7 +98,7 @@ public void refCountedXdsClientObjectPool_delayedCreation() { BootstrapInfo bootstrapInfo = BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build(); RefCountedXdsClientObjectPool xdsClientPool = - new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET, new MetricRecorder() {}); + new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET, metricRecorder); assertThat(xdsClientPool.getXdsClientForTest()).isNull(); XdsClient xdsClient = xdsClientPool.getObject(); assertThat(xdsClientPool.getXdsClientForTest()).isNotNull(); @@ -105,7 +111,7 @@ public void refCountedXdsClientObjectPool_refCounted() { BootstrapInfo bootstrapInfo = BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build(); RefCountedXdsClientObjectPool xdsClientPool = - new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET, new MetricRecorder() {}); + new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET, metricRecorder); // getObject once XdsClient xdsClient = xdsClientPool.getObject(); assertThat(xdsClient).isNotNull(); @@ -125,7 +131,7 @@ public void refCountedXdsClientObjectPool_getObjectCreatesNewInstanceIfAlreadySh BootstrapInfo bootstrapInfo = BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build(); RefCountedXdsClientObjectPool xdsClientPool = - new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET, new MetricRecorder() {}); + new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET, metricRecorder); XdsClient xdsClient1 = xdsClientPool.getObject(); assertThat(xdsClientPool.returnObject(xdsClient1)).isNull(); assertThat(xdsClient1.isShutDown()).isTrue(); diff --git a/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java b/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java index a42b0129db0..4ba2b64d764 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java @@ -20,6 +20,7 @@ import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -31,15 +32,12 @@ import io.grpc.MetricRecorder; import io.grpc.MetricRecorder.BatchCallback; import io.grpc.MetricRecorder.BatchRecorder; -import io.grpc.MetricRecorder.Registration; -import io.grpc.testing.GrpcCleanupRule; +import io.grpc.MetricSink; import io.grpc.xds.XdsClientMetricReporter.CallbackMetricReporter; +import io.grpc.xds.XdsClientMetricReporterImpl.CallbackMetricReporterImpl; import io.grpc.xds.client.XdsClient; import java.util.Arrays; import java.util.Collections; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeoutException; -import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -73,8 +71,6 @@ public class XdsClientMetricReporterImplTest { private XdsClient mockXdsClient; @Mock private BatchRecorder mockBatchRecorder; - @Mock - private Registration mockGaugeRegistration; @Captor private ArgumentCaptor gaugeBatchCallbackCaptor; @@ -82,20 +78,31 @@ public class XdsClientMetricReporterImplTest { @Before public void setUp() { - when(mockMetricRecorder.registerBatchCallback(any(), any())).thenReturn(mockGaugeRegistration); reporter = new XdsClientMetricReporterImpl(mockMetricRecorder); } @Test public void reportResourceUpdates() { + // TODO(dnvindhya): support the "authority" label once available. reporter.reportResourceUpdates(10, 5, target, server, resourceTypeUrl); - verifyValidInvalidResourceCounterAdd(10, 5, target, server, resourceTypeUrl); + verify(mockMetricRecorder).addLongCounter( + eqMetricInstrumentName("grpc.xds_client.resource_updates_valid"), eq((long) 10), + eq(Lists.newArrayList(target, server, resourceTypeUrl)), + eq(Lists.newArrayList())); + verify(mockMetricRecorder).addLongCounter( + eqMetricInstrumentName("grpc.xds_client.resource_updates_invalid"), + eq((long) 5), + eq(Lists.newArrayList(target, server, resourceTypeUrl)), + eq(Lists.newArrayList())); } @Test public void reportServerFailure() { reporter.reportServerFailure(1, target, server); - verifyServerFailureCounterAdd("grpc.xds_client.server_failure", 1, target, server); + verify(mockMetricRecorder).addLongCounter( + eqMetricInstrumentName("grpc.xds_client.server_failure"), eq((long) 1), + eq(Lists.newArrayList(target, server)), + eq(Lists.newArrayList())); } @Test @@ -107,61 +114,62 @@ public void setXdsClient_reportMetrics() throws Exception { when(mockXdsClient.reportServerConnections(any(CallbackMetricReporter.class))) .thenReturn(future); reporter.setXdsClient(mockXdsClient); - verify(mockMetricRecorder).registerBatchCallback(any(MetricRecorder.BatchCallback.class), + verify(mockMetricRecorder).registerBatchCallback(gaugeBatchCallbackCaptor.capture(), eqMetricInstrumentName("grpc.xds_client.connected"), eqMetricInstrumentName("grpc.xds_client.resources")); - - reporter.reportCallbackMetrics(mockBatchRecorder); + gaugeBatchCallbackCaptor.getValue().accept(mockBatchRecorder); verify(mockXdsClient).reportResourceCounts(any(CallbackMetricReporter.class)); verify(mockXdsClient).reportServerConnections(any(CallbackMetricReporter.class)); } @Test - public void setXdsClient_reportMetrics_exception() throws Exception { - SettableFuture future = SettableFuture.create(); - future.setException(new Exception("test")); - when(mockXdsClient.reportResourceCounts(any(CallbackMetricReporter.class))) - .thenReturn(future); + public void setXdsClient_reportCallbackMetrics_resourceCountsFails() { + SettableFuture resourceCountsFuture = SettableFuture.create(); + resourceCountsFuture.setException(new Exception("test")); + when(mockXdsClient.reportResourceCounts(any())).thenReturn(resourceCountsFuture); + reporter.setXdsClient(mockXdsClient); - verify(mockMetricRecorder).registerBatchCallback(any(MetricRecorder.BatchCallback.class), - eqMetricInstrumentName("grpc.xds_client.connected"), - eqMetricInstrumentName("grpc.xds_client.resources")); + verify(mockMetricRecorder) + .registerBatchCallback(gaugeBatchCallbackCaptor.capture(), any(), any()); + gaugeBatchCallbackCaptor.getValue().accept(mockBatchRecorder); - reporter.reportCallbackMetrics(mockBatchRecorder); verify(mockXdsClient).reportResourceCounts(any(CallbackMetricReporter.class)); - verify(mockXdsClient, never()).reportServerConnections( - any(CallbackMetricReporter.class)); + verify(mockXdsClient, never()).reportServerConnections(any(CallbackMetricReporter.class)); } - // @Test - // public void metricGauges() throws ExecutionException, InterruptedException, TimeoutException { - // reporter.setXdsClient(mockXdsClient); - // verify(mockMetricRecorder).registerBatchCallback(gaugeBatchCallbackCaptor.capture(), - // any()); - // BatchCallback gaugeBatchCallback = gaugeBatchCallbackCaptor.getValue(); - // // Verify the correct resource gauge values when requested at this point. - // InOrder inOrder = inOrder(mockBatchRecorder); - // gaugeBatchCallback.accept(mockBatchRecorder); - // - // verify(mockXdsClient).reportResourceCounts(any(CallbackMetricReporter.class)); - // verify(mockXdsClient).reportServerConnections(any(CallbackMetricReporter.class)); - // - // inOrder.verify(mockBatchRecorder).recordLongGauge( - // argThat(new LongGaugeInstrumentArgumentMatcher("grpc.lb.rls.cache_entries")), eq(0L), - // any(), any()); - // inOrder.verify(mockBatchRecorder) - // .recordLongGauge(argThat(new LongGaugeInstrumentArgumentMatcher( - // "grpc.lb.rls.cache_size")), - // eq(0L), any(), any()); - // } - @Test - public void close() { + public void metricGauges() { + SettableFuture future = SettableFuture.create(); + future.set(null); + when(mockXdsClient.reportResourceCounts(any(CallbackMetricReporter.class))) + .thenReturn(future); + when(mockXdsClient.reportServerConnections(any(CallbackMetricReporter.class))) + .thenReturn(future); + CallbackMetricReporterImpl callbackMetricReporter = new CallbackMetricReporterImpl( + mockBatchRecorder); + reporter.injectCallbackMetricReporter(callbackMetricReporter); reporter.setXdsClient(mockXdsClient); - verify(mockMetricRecorder).registerBatchCallback(any(MetricRecorder.BatchCallback.class), + verify(mockMetricRecorder).registerBatchCallback(gaugeBatchCallbackCaptor.capture(), eqMetricInstrumentName("grpc.xds_client.connected"), eqMetricInstrumentName("grpc.xds_client.resources")); - reporter.close(); + BatchCallback gaugeBatchCallback = gaugeBatchCallbackCaptor.getValue(); + // Verify the correct resource gauge values when requested at this point. + InOrder inOrder = inOrder(mockBatchRecorder); + gaugeBatchCallback.accept(mockBatchRecorder); + + verify(mockXdsClient).reportResourceCounts(eq(callbackMetricReporter)); + verify(mockXdsClient).reportServerConnections(eq(callbackMetricReporter)); + + callbackMetricReporter.reportResourceCounts(10, "acked", resourceTypeUrl, target); + inOrder.verify(mockBatchRecorder) + .recordLongGauge(eqMetricInstrumentName("grpc.xds_client.resources"), eq(10L), any(), + any()); + + callbackMetricReporter.reportServerConnections(1, target, "xdsServer"); + inOrder.verify(mockBatchRecorder) + .recordLongGauge(eqMetricInstrumentName("grpc.xds_client.connected"), eq(1L), any(), any()); + + inOrder.verifyNoMoreInteractions(); } @Test @@ -183,28 +191,18 @@ public void callbackMetricReporter() { eq(Collections.emptyList())); } - private void verifyServerFailureCounterAdd(String name, long value, - String dataPlaneTargetLabel, String xdsServer) { - verify(mockMetricRecorder).addLongCounter( - eqMetricInstrumentName(name), eq(value), - eq(Lists.newArrayList(dataPlaneTargetLabel, xdsServer)), - eq(Lists.newArrayList())); - } + @Test + public void close_closesGaugeRegistration() { + MetricSink.Registration mockRegistration = mock(MetricSink.Registration.class); + when(mockMetricRecorder.registerBatchCallback(any(MetricRecorder.BatchCallback.class), + eqMetricInstrumentName("grpc.xds_client.connected"), + eqMetricInstrumentName("grpc.xds_client.resources"))).thenReturn(mockRegistration); - private void verifyValidInvalidResourceCounterAdd(long validResourceCount, - long invalidResourceCount, - String target, String xdsServer, String resourceTypeUrl) { - // TODO(dnvindhya): support the "authority" label once available. - verify(mockMetricRecorder).addLongCounter( - eqMetricInstrumentName("grpc.xds_client.resource_updates_valid"), eq(validResourceCount), - eq(Lists.newArrayList(target, xdsServer, resourceTypeUrl)), - eq(Lists.newArrayList())); - // TODO(dnvindhya): support the "authority" label once available. - verify(mockMetricRecorder).addLongCounter( - eqMetricInstrumentName("grpc.xds_client.resource_updates_invalid"), - eq(invalidResourceCount), - eq(Lists.newArrayList(target, xdsServer, resourceTypeUrl)), - eq(Lists.newArrayList())); + // Sets XdsClient and register the gauges + reporter.setXdsClient(mockXdsClient); + // Closes registered gauges + reporter.close(); + verify(mockRegistration, times(1)).close(); } @SuppressWarnings("TypeParameterUnusedInFormals") diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java index 23c42d85409..79aa748ed6e 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java @@ -153,6 +153,7 @@ public ConfigOrError parseServiceConfig(Map rawServiceConfig) { private final CallInfo call1 = new CallInfo("HelloService", "hi"); private final CallInfo call2 = new CallInfo("GreetService", "bye"); private final TestChannel channel = new TestChannel(); + private final MetricRecorder metricRecorder = new MetricRecorder() {}; private BootstrapInfo bootstrapInfo = BootstrapInfo.builder() .servers(ImmutableList.of(ServerInfo.create( "td.googleapis.com", InsecureChannelCredentials.create()))) @@ -188,7 +189,7 @@ public void setUp() { RouterFilter.INSTANCE); resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, filterRegistry, null, new MetricRecorder() {}); + xdsClientPoolFactory, mockRandom, filterRegistry, null, metricRecorder); } @After @@ -235,7 +236,7 @@ public List getTargets() { resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, serviceConfigParser, syncContext, scheduler, xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, - new MetricRecorder() {}); + metricRecorder); resolver.start(mockListener); verify(mockListener).onError(errorCaptor.capture()); Status error = errorCaptor.getValue(); @@ -249,7 +250,7 @@ public void resolving_withTargetAuthorityNotFound() { resolver = new XdsNameResolver(targetUri, "notfound.google.com", AUTHORITY, null, serviceConfigParser, syncContext, scheduler, xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, - new MetricRecorder() {}); + metricRecorder); resolver.start(mockListener); verify(mockListener).onError(errorCaptor.capture()); Status error = errorCaptor.getValue(); @@ -271,7 +272,7 @@ public void resolving_noTargetAuthority_templateWithoutXdstp() { resolver = new XdsNameResolver( targetUri, null, serviceAuthority, null, serviceConfigParser, syncContext, scheduler, xdsClientPoolFactory, - mockRandom, FilterRegistry.getDefaultRegistry(), null, new MetricRecorder() {}); + mockRandom, FilterRegistry.getDefaultRegistry(), null, metricRecorder); resolver.start(mockListener); verify(mockListener, never()).onError(any(Status.class)); } @@ -292,7 +293,7 @@ public void resolving_noTargetAuthority_templateWithXdstp() { resolver = new XdsNameResolver( targetUri, null, serviceAuthority, null, serviceConfigParser, syncContext, scheduler, xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, - new MetricRecorder() {}); + metricRecorder); resolver.start(mockListener); verify(mockListener, never()).onError(any(Status.class)); } @@ -313,7 +314,7 @@ public void resolving_noTargetAuthority_xdstpWithMultipleSlashes() { resolver = new XdsNameResolver( targetUri, null, serviceAuthority, null, serviceConfigParser, syncContext, scheduler, xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, - new MetricRecorder() {}); + metricRecorder); // The Service Authority must be URL encoded, but unlike the LDS resource name. @@ -342,7 +343,7 @@ public void resolving_targetAuthorityInAuthoritiesMap() { resolver = new XdsNameResolver(targetUri, "xds.authority.com", serviceAuthority, null, serviceConfigParser, syncContext, scheduler, xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, - new MetricRecorder() {}); + metricRecorder); resolver.start(mockListener); verify(mockListener, never()).onError(any(Status.class)); } @@ -375,7 +376,7 @@ public void resolving_ldsResourceUpdateRdsName() { resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, serviceConfigParser, syncContext, scheduler, xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, - new MetricRecorder() {}); + metricRecorder); // use different ldsResourceName and service authority. The virtualhost lookup should use // service authority. expectedLdsResourceName = "test-" + expectedLdsResourceName; @@ -557,7 +558,7 @@ public void resolving_matchingVirtualHostNotFound_matchingOverrideAuthority() { resolver = new XdsNameResolver(targetUri, null, AUTHORITY, "random", serviceConfigParser, syncContext, scheduler, xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, - new MetricRecorder() {}); + metricRecorder); resolver.start(mockListener); FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); xdsClient.deliverLdsUpdate(0L, Arrays.asList(virtualHost)); @@ -581,7 +582,7 @@ public void resolving_matchingVirtualHostNotFound_notMatchingOverrideAuthority() resolver = new XdsNameResolver(targetUri, null, AUTHORITY, "random", serviceConfigParser, syncContext, scheduler, xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, - new MetricRecorder() {}); + metricRecorder); resolver.start(mockListener); FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); xdsClient.deliverLdsUpdate(0L, Arrays.asList(virtualHost)); @@ -593,7 +594,7 @@ public void resolving_matchingVirtualHostNotFoundForOverrideAuthority() { resolver = new XdsNameResolver(targetUri, null, AUTHORITY, AUTHORITY, serviceConfigParser, syncContext, scheduler, xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, - new MetricRecorder() {}); + metricRecorder); resolver.start(mockListener); FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); xdsClient.deliverLdsUpdate(0L, buildUnmatchedVirtualHosts()); @@ -678,7 +679,7 @@ public void retryPolicyInPerMethodConfigGeneratedByResolverIsValid() { true, 5, 5, new AutoConfiguredLoadBalancerFactory("pick-first")); resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, realParser, syncContext, scheduler, xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, - new MetricRecorder() {}); + metricRecorder); resolver.start(mockListener); FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); RetryPolicy retryPolicy = RetryPolicy.create( @@ -889,7 +890,7 @@ public void resolved_rpcHashingByChannelId() { resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, serviceConfigParser, syncContext, scheduler, xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, - new MetricRecorder() {}); + metricRecorder); resolver.start(mockListener); xdsClient = (FakeXdsClient) resolver.getXdsClient(); xdsClient.deliverLdsUpdate( @@ -2013,7 +2014,7 @@ public ObjectPool get(String target) { @Override public ObjectPool getOrCreate(String target) throws XdsInitializationException { - return getOrCreate(target, new MetricRecorder() {}); + return getOrCreate(target, metricRecorder); } @Override From f60c98504c29b508ec9c14986f8417ab8144d82f Mon Sep 17 00:00:00 2001 From: Vindhya Ningegowda Date: Fri, 1 Nov 2024 19:09:52 -0700 Subject: [PATCH 03/13] fix javadoc --- api/src/main/java/io/grpc/NameResolver.java | 2 - .../io/grpc/xds/XdsClientMetricReporter.java | 52 ++++++++----------- .../grpc/xds/XdsClientMetricReporterImpl.java | 27 +++------- .../java/io/grpc/xds/client/XdsClient.java | 17 +++++- .../io/grpc/xds/client/XdsClientImpl.java | 2 +- .../grpc/xds/GrpcXdsClientImplTestBase.java | 5 +- .../xds/XdsClientMetricReporterImplTest.java | 2 +- .../java/io/grpc/xds/XdsNameResolverTest.java | 4 +- 8 files changed, 52 insertions(+), 59 deletions(-) diff --git a/api/src/main/java/io/grpc/NameResolver.java b/api/src/main/java/io/grpc/NameResolver.java index 8ed28730015..b7fcb3089b8 100644 --- a/api/src/main/java/io/grpc/NameResolver.java +++ b/api/src/main/java/io/grpc/NameResolver.java @@ -408,8 +408,6 @@ public String getOverrideAuthority() { /** * Returns the {@link MetricRecorder} that the channel uses to record metrics. - * - * @since 1.67.0 */ @Nullable @ExperimentalApi("Insert GitHub issue") diff --git a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporter.java b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporter.java index 09113f12f06..de5fafcf058 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporter.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporter.java @@ -21,38 +21,35 @@ /** * Interface for reporting metrics from the xDS client. - * We need this indirection, to de couple Xds from OpenTelemetry */ @Internal public interface XdsClientMetricReporter { /** - * Reports resource update counts. + * Reports number of valid and invalid resources. * - * @param validResourceCount the number of resources that were successfully updated. - * @param invalidResourceCount the number of resources that failed to update. - * @param target the xDS management server name for the load balancing policy this update is for. - * @param xdsServer the xDS management server address for this update. - * @param resourceType the type of resource (e.g., "LDS", "RDS", "CDS", "EDS"). + * @param validResourceCount Number of resources that were valid. + * @param invalidResourceCount Number of resources that were invalid. + * @param target Target of the gRPC channel. + * @param xdsServer Target URI of the xDS server with which the XdsClient is communicating. + * @param resourceType Type of XDS resource (e.g., "envoy.config.listener.v3.Listener"). */ default void reportResourceUpdates(long validResourceCount, long invalidResourceCount, String target, String xdsServer, String resourceType) { } /** - * Reports xDS server failure counts. + * Reports number of xDS servers going from healthy to unhealthy. * - * @param serverFailure the number of times the xDS server has failed. - * @param target the xDS management server name for the load balancing policy this failure is for. - * @param xdsServer the xDS management server address for this failure. + * @param serverFailure Number of xDS server failures. + * @param target Target of the gRPC channel. + * @param xdsServer Target URI of the xDS server with which the XdsClient is communicating. */ default void reportServerFailure(long serverFailure, String target, String xdsServer) { } /** - * Sets the {@link XdsClient} instance to the reporter. - * - * @param xdsClient the {@link XdsClient} instance. + * Sets the {@link XdsClient} instance. */ default void setXdsClient(XdsClient xdsClient) { } @@ -64,34 +61,31 @@ default void close() { } /** - * Interface for reporting metrics from the xDS client callbacks. + * Interface for reporting metrics through callback. * */ interface CallbackMetricReporter { /** - * Reports resource counts in the cache. + * Reports number of resources in each cache state. * - * @param resourceCount the number of resources in the cache. - * @param cacheState the state of the cache (e.g., "SYNCED", "DOES_NOT_EXIST"). - * @param resourceType the type of resource (e.g., "LDS", "RDS", "CDS", "EDS"). - * @param target the xDS management server name for the load balancing policy this count is - * for. + * @param resourceCount Number of resources. + * @param cacheState Status of the resource metadata + * {@link io.grpc.xds.client.XdsClient.ResourceMetadata.ResourceMetadataStatus}. + * @param resourceType Type of XDS resource (e.g., "envoy.config.listener.v3.Listener"). + * @param target Target of the gRPC channel. */ - // TODO(@dnvindhya): include the "authority" label once authority is available. + // TODO(@dnvindhya): include the "authority" label once xds.authority is available. default void reportResourceCounts(long resourceCount, String cacheState, String resourceType, String target) { } /** - * Reports server connection status. + * Reports whether xDS client has a working ADS stream to the xDS server. * - * @param isConnected {@code true} if the client is connected to the xDS server, {@code false} - * otherwise. - * @param target the xDS management server name for the load balancing policy this connection - * is for. - * @param xdsServer the xDS management server address for this connection. - * @since 0.1.0 + * @param isConnected 1 if the client is connected to the xDS server, 0 otherwise. + * @param target Target of the gRPC channel. + * @param xdsServer Target URI of the xDS server with which the XdsClient is communicating. */ default void reportServerConnections(int isConnected, String target, String xdsServer) { } diff --git a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java index 5a48ee86643..5e6ea7f6aaf 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java @@ -108,13 +108,6 @@ public void reportServerFailure(long serverFailure, String target, String xdsSer Arrays.asList(target, xdsServer), Collections.emptyList()); } - @Override - public void close() { - if (gaugeRegistration != null) { - gaugeRegistration.close(); - } - } - @Override public void setXdsClient(XdsClient client) { this.xdsClient = client; @@ -127,9 +120,13 @@ public void accept(BatchRecorder recorder) { }, CONNECTED_GAUGE, RESOURCES_GAUGE); } - // ... (other methods) + @Override + public void close() { + if (gaugeRegistration != null) { + gaugeRegistration.close(); + } + } - @VisibleForTesting void reportCallbackMetrics(BatchRecorder recorder) { if (callbackMetricReporter == null) { // Instantiate only if not injected @@ -146,11 +143,6 @@ void reportCallbackMetrics(BatchRecorder recorder) { } } - /** - * Reports resource counts. - * This method is extracted for better testability. - */ - @VisibleForTesting void reportResourceCounts(CallbackMetricReporter callbackMetricReporter) throws Exception { SettableFuture ret = this.xdsClient.reportResourceCounts( callbackMetricReporter); @@ -158,11 +150,6 @@ void reportResourceCounts(CallbackMetricReporter callbackMetricReporter) throws Void unused = ret.get(5, TimeUnit.SECONDS); } - /** - * Reports server connections. - * This method is extracted for better testability. - */ - @VisibleForTesting void reportServerConnections(CallbackMetricReporter callbackMetricReporter) throws Exception { SettableFuture ret = this.xdsClient.reportServerConnections(callbackMetricReporter); // Normally this shouldn't take long, but adding a timeout to avoid indefinite blocking @@ -192,7 +179,7 @@ public void reportServerConnections(int isConnected, String target, String xdsSe Collections.emptyList()); } - // TODO(@dnvindhya): include the "authority" label once authority is available. + // TODO(@dnvindhya): include the "authority" label once xds.authority is available. @Override public void reportResourceCounts(long resourceCount, String cacheState, String resourceType, String target) { diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClient.java b/xds/src/main/java/io/grpc/xds/client/XdsClient.java index 0e9c2341f50..46d82656b12 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClient.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClient.java @@ -380,11 +380,24 @@ public Map getServerLrsClientMap() { throw new UnsupportedOperationException(); } - public SettableFuture reportResourceCounts( - CallbackMetricReporter callbackMetricReporter) { + /** + * Reports the number of resources in each cache state through {@link CallbackMetricReporter}. + * + *

Cache state is determined by two factors: + *

    + *
  • Whether the resource is cached. + *
  • The {@link io.grpc.xds.client.XdsClient.ResourceMetadata.ResourceMetadataStatus} of the + * resource. + *
+ */ + public SettableFuture reportResourceCounts(CallbackMetricReporter callbackMetricReporter) { throw new UnsupportedOperationException(); } + /** + * Reports whether xDS client has a working ADS stream to xDS server. Reporting is done through + * {@link CallbackMetricReporter}. + */ public SettableFuture reportServerConnections( CallbackMetricReporter callbackMetricReporter) { throw new UnsupportedOperationException(); diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java index 88eae965e4e..b8bdbbafc33 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java @@ -571,7 +571,7 @@ private String cacheStateFromResourceStatus(ResourceMetadata metadata, boolean i } /** - * Calculates number of resources by ResourceType and ResourceSubscriber.metadata.status + * Calculates number of resources by ResourceType and ResourceSubscriber.metadata.status. */ Map, Map> getResourceCountsByType() { Map, Map> resourceCountsByType = new HashMap<>(); diff --git a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java index 994e1bd747d..4f59f715b46 100644 --- a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java +++ b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java @@ -2575,10 +2575,11 @@ public void cdsResponseErrorHandling_badUpstreamTlsContext() { call.sendResponse(CDS, clusters, VERSION_1, "0000"); // The response NACKed with errors indicating indices of the failed resources. - String errorMsg = "CDS response Cluster 'cluster.googleapis.com' validation error: " + String errorMsg = "CDS response Cluster 'cluster.googleapis.com' validation error: " + "Cluster cluster.googleapis.com: malformed UpstreamTlsContext: " + "io.grpc.xds.client.XdsResourceType$ResourceInvalidException: " - + "ca_certificate_provider_instance is required in upstream-tls-context"; + + "ca_certificate_provider_instance or system_root_certs is required in " + + "upstream-tls-context"; call.verifyRequestNack(CDS, CDS_RESOURCE, "", "0000", NODE, ImmutableList.of(errorMsg)); verify(cdsResourceWatcher).onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, errorMsg); diff --git a/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java b/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java index 4ba2b64d764..05c043955fa 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java @@ -83,7 +83,7 @@ public void setUp() { @Test public void reportResourceUpdates() { - // TODO(dnvindhya): support the "authority" label once available. + // TODO(dnvindhya): add the "authority" label once available. reporter.reportResourceUpdates(10, 5, target, server, resourceTypeUrl); verify(mockMetricRecorder).addLongCounter( eqMetricInstrumentName("grpc.xds_client.resource_updates_valid"), eq((long) 10), diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java index 79aa748ed6e..71dd8fadccf 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java @@ -923,7 +923,7 @@ public void resolved_rpcHashingByChannelId() { public void resolved_routeActionHasAutoHostRewrite_emitsCallOptionForTheSame() { resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, serviceConfigParser, syncContext, scheduler, xdsClientPoolFactory, mockRandom, - FilterRegistry.getDefaultRegistry(), null); + FilterRegistry.getDefaultRegistry(), null, metricRecorder); resolver.start(mockListener); FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); xdsClient.deliverLdsUpdate( @@ -954,7 +954,7 @@ public void resolved_routeActionHasAutoHostRewrite_emitsCallOptionForTheSame() { public void resolved_routeActionNoAutoHostRewrite_doesntEmitCallOptionForTheSame() { resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, serviceConfigParser, syncContext, scheduler, xdsClientPoolFactory, mockRandom, - FilterRegistry.getDefaultRegistry(), null); + FilterRegistry.getDefaultRegistry(), null, metricRecorder); resolver.start(mockListener); FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); xdsClient.deliverLdsUpdate( From 4f01651765deef86756695392b0eb6d432c79d08 Mon Sep 17 00:00:00 2001 From: Vindhya Ningegowda Date: Thu, 7 Nov 2024 18:29:46 -0800 Subject: [PATCH 04/13] Instead of storing MetricRecorder, pass it as is for XdsClient creation --- api/src/main/java/io/grpc/NameResolver.java | 2 - .../grpc/xds/SharedXdsClientPoolProvider.java | 32 +++++---------- .../grpc/xds/XdsClientMetricReporterImpl.java | 40 ++++++------------- .../io/grpc/xds/XdsClientPoolFactory.java | 2 - .../java/io/grpc/xds/XdsServerWrapper.java | 3 +- .../java/io/grpc/xds/client/XdsClient.java | 2 +- .../io/grpc/xds/client/XdsClientImpl.java | 6 +-- .../{ => client}/XdsClientMetricReporter.java | 33 ++------------- .../java/io/grpc/xds/CsdsServiceTest.java | 4 -- .../grpc/xds/GrpcXdsClientImplTestBase.java | 1 + .../xds/SharedXdsClientPoolProviderTest.java | 20 +++++----- .../io/grpc/xds/XdsClientFederationTest.java | 4 +- .../xds/XdsClientMetricReporterImplTest.java | 12 ++++-- .../java/io/grpc/xds/XdsNameResolverTest.java | 10 ----- .../java/io/grpc/xds/XdsServerTestHelper.java | 5 --- 15 files changed, 51 insertions(+), 125 deletions(-) rename xds/src/main/java/io/grpc/xds/{ => client}/XdsClientMetricReporter.java (68%) diff --git a/api/src/main/java/io/grpc/NameResolver.java b/api/src/main/java/io/grpc/NameResolver.java index b7fcb3089b8..94e7a02ce15 100644 --- a/api/src/main/java/io/grpc/NameResolver.java +++ b/api/src/main/java/io/grpc/NameResolver.java @@ -410,7 +410,6 @@ public String getOverrideAuthority() { * Returns the {@link MetricRecorder} that the channel uses to record metrics. */ @Nullable - @ExperimentalApi("Insert GitHub issue") public MetricRecorder getMetricRecorder() { return metricRecorder; } @@ -563,7 +562,6 @@ public Builder setOverrideAuthority(String authority) { /** * See {@link Args#getMetricRecorder()}. This is an optional field. */ - @ExperimentalApi("Insert github issue") public Builder setMetricRecorder(MetricRecorder metricRecorder) { this.metricRecorder = metricRecorder; return this; diff --git a/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java b/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java index 2cc4e8d917a..2ab1d995aac 100644 --- a/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java +++ b/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java @@ -57,7 +57,6 @@ final class SharedXdsClientPoolProvider implements XdsClientPoolFactory { private final Object lock = new Object(); private final AtomicReference> bootstrapOverride = new AtomicReference<>(); private final Map> targetToXdsClientMap = new ConcurrentHashMap<>(); - private final Map targetToMetricRecorderMap = new ConcurrentHashMap<>(); SharedXdsClientPoolProvider() { this(new GrpcBootstrapperImpl()); @@ -83,12 +82,6 @@ public ObjectPool get(String target) { return targetToXdsClientMap.get(target); } - @Override - @Nullable - public ObjectPool getOrCreate(String target) throws XdsInitializationException { - return this.getOrCreate(target, new MetricRecorder() {}); - } - @Override public ObjectPool getOrCreate(String target, MetricRecorder metricRecorder) throws XdsInitializationException { @@ -107,12 +100,8 @@ public ObjectPool getOrCreate(String target, MetricRecorder metricRec if (bootstrapInfo.servers().isEmpty()) { throw new XdsInitializationException("No xDS server provided"); } - MetricRecorder metricRecorderForTarget = targetToMetricRecorderMap.get(target); - metricRecorder = - metricRecorderForTarget != null ? metricRecorderForTarget : metricRecorder; ref = new RefCountedXdsClientObjectPool(bootstrapInfo, target, metricRecorder); targetToXdsClientMap.put(target, ref); - targetToMetricRecorderMap.putIfAbsent(target, metricRecorder); } } } @@ -124,25 +113,19 @@ public ImmutableList getTargets() { return ImmutableList.copyOf(targetToXdsClientMap.keySet()); } - @VisibleForTesting - MetricRecorder getMetricRecorder(String target) { - return targetToMetricRecorderMap.get(target); - } - - private static class SharedXdsClientPoolProviderHolder { private static final SharedXdsClientPoolProvider instance = new SharedXdsClientPoolProvider(); } @ThreadSafe @VisibleForTesting - static class RefCountedXdsClientObjectPool implements ObjectPool { + protected class RefCountedXdsClientObjectPool implements ObjectPool { - private static final ExponentialBackoffPolicy.Provider BACKOFF_POLICY_PROVIDER = + private final ExponentialBackoffPolicy.Provider backoffPolicyProvider = new ExponentialBackoffPolicy.Provider(); private final BootstrapInfo bootstrapInfo; private final String target; // The target associated with the xDS client. - private final MetricRecorder metricRecorder; + private final XdsClientMetricReporterImpl xdsClientMetricReporter; private final Object lock = new Object(); @GuardedBy("lock") private ScheduledExecutorService scheduler; @@ -156,7 +139,7 @@ static class RefCountedXdsClientObjectPool implements ObjectPool { MetricRecorder metricRecorder) { this.bootstrapInfo = checkNotNull(bootstrapInfo); this.target = target; - this.metricRecorder = metricRecorder; + this.xdsClientMetricReporter = new XdsClientMetricReporterImpl(metricRecorder); } @Override @@ -171,13 +154,14 @@ public XdsClient getObject() { DEFAULT_XDS_TRANSPORT_FACTORY, bootstrapInfo, scheduler, - BACKOFF_POLICY_PROVIDER, + backoffPolicyProvider, GrpcUtil.STOPWATCH_SUPPLIER, TimeProvider.SYSTEM_TIME_PROVIDER, MessagePrinter.INSTANCE, new TlsContextManagerImpl(bootstrapInfo), getTarget(), - new XdsClientMetricReporterImpl(metricRecorder)); + xdsClientMetricReporter); + xdsClientMetricReporter.setXdsClient(xdsClient); } refCount++; return xdsClient; @@ -191,6 +175,8 @@ public XdsClient returnObject(Object object) { if (refCount == 0) { xdsClient.shutdown(); xdsClient = null; + xdsClientMetricReporter.close(); + targetToXdsClientMap.remove(target); scheduler = SharedResourceHolder.release(GrpcUtil.TIMER_SERVICE, scheduler); } return null; diff --git a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java index 5e6ea7f6aaf..535efacd3d3 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java @@ -27,6 +27,7 @@ import io.grpc.MetricRecorder.BatchRecorder; import io.grpc.MetricRecorder.Registration; import io.grpc.xds.client.XdsClient; +import io.grpc.xds.client.XdsClientMetricReporter; import java.util.Arrays; import java.util.Collections; import java.util.concurrent.TimeUnit; @@ -53,8 +54,6 @@ public class XdsClientMetricReporterImpl implements XdsClientMetricReporter { private Registration gaugeRegistration = null; @Nullable private XdsClient xdsClient = null; - @Nullable - private CallbackMetricReporter callbackMetricReporter = null; static { MetricInstrumentRegistry metricInstrumentRegistry @@ -108,8 +107,7 @@ public void reportServerFailure(long serverFailure, String target, String xdsSer Arrays.asList(target, xdsServer), Collections.emptyList()); } - @Override - public void setXdsClient(XdsClient client) { + void setXdsClient(XdsClient client) { this.xdsClient = client; // register gauge here this.gaugeRegistration = metricRecorder.registerBatchCallback(new BatchCallback() { @@ -120,21 +118,22 @@ public void accept(BatchRecorder recorder) { }, CONNECTED_GAUGE, RESOURCES_GAUGE); } - @Override - public void close() { + void close() { if (gaugeRegistration != null) { gaugeRegistration.close(); } } void reportCallbackMetrics(BatchRecorder recorder) { - if (callbackMetricReporter == null) { - // Instantiate only if not injected - callbackMetricReporter = new CallbackMetricReporterImpl(recorder); - } + CallbackMetricReporter callbackMetricReporter = createCallbackMetricReporter(recorder); try { - reportResourceCounts(callbackMetricReporter); - reportServerConnections(callbackMetricReporter); + SettableFuture ret = this.xdsClient.reportResourceCounts( + callbackMetricReporter); + // Normally this shouldn't take long, but adding a timeout to avoid indefinite blocking + Void unused = ret.get(5, TimeUnit.SECONDS); + SettableFuture ret1 = this.xdsClient.reportServerConnections(callbackMetricReporter); + // Normally this shouldn't take long, but adding a timeout to avoid indefinite blocking + Void unused1 = ret1.get(5, TimeUnit.SECONDS); } catch (Exception e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); // re-set the current thread's interruption state @@ -143,25 +142,12 @@ void reportCallbackMetrics(BatchRecorder recorder) { } } - void reportResourceCounts(CallbackMetricReporter callbackMetricReporter) throws Exception { - SettableFuture ret = this.xdsClient.reportResourceCounts( - callbackMetricReporter); - // Normally this shouldn't take long, but adding a timeout to avoid indefinite blocking - Void unused = ret.get(5, TimeUnit.SECONDS); - } - - void reportServerConnections(CallbackMetricReporter callbackMetricReporter) throws Exception { - SettableFuture ret = this.xdsClient.reportServerConnections(callbackMetricReporter); - // Normally this shouldn't take long, but adding a timeout to avoid indefinite blocking - Void unused = ret.get(5, TimeUnit.SECONDS); - } - /** * Allows injecting a custom {@link CallbackMetricReporter} for testing purposes. */ @VisibleForTesting - void injectCallbackMetricReporter(CallbackMetricReporter reporter) { - this.callbackMetricReporter = reporter; + CallbackMetricReporter createCallbackMetricReporter(BatchRecorder recorder) { + return new CallbackMetricReporterImpl(recorder); } @VisibleForTesting diff --git a/xds/src/main/java/io/grpc/xds/XdsClientPoolFactory.java b/xds/src/main/java/io/grpc/xds/XdsClientPoolFactory.java index ec2dc373d57..f10d6504d79 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientPoolFactory.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientPoolFactory.java @@ -30,8 +30,6 @@ interface XdsClientPoolFactory { @Nullable ObjectPool get(String target); - ObjectPool getOrCreate(String target) throws XdsInitializationException; - ObjectPool getOrCreate(String target, MetricRecorder metricRecorder) throws XdsInitializationException; diff --git a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java index bd622a71124..e1387935314 100644 --- a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java +++ b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java @@ -29,6 +29,7 @@ import io.grpc.InternalServerInterceptors; import io.grpc.Metadata; import io.grpc.MethodDescriptor; +import io.grpc.MetricRecorder; import io.grpc.Server; import io.grpc.ServerBuilder; import io.grpc.ServerCall; @@ -171,7 +172,7 @@ public void run() { private void internalStart() { try { - xdsClientPool = xdsClientPoolFactory.getOrCreate(""); + xdsClientPool = xdsClientPoolFactory.getOrCreate("", new MetricRecorder() {}); } catch (Exception e) { StatusException statusException = Status.UNAVAILABLE.withDescription( "Failed to initialize xDS").withCause(e).asException(); diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClient.java b/xds/src/main/java/io/grpc/xds/client/XdsClient.java index 46d82656b12..428778fb216 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClient.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClient.java @@ -28,8 +28,8 @@ import com.google.protobuf.Any; import io.grpc.ExperimentalApi; import io.grpc.Status; -import io.grpc.xds.XdsClientMetricReporter.CallbackMetricReporter; import io.grpc.xds.client.Bootstrapper.ServerInfo; +import io.grpc.xds.client.XdsClientMetricReporter.CallbackMetricReporter; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java index b8bdbbafc33..f4244ff8354 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java @@ -38,12 +38,12 @@ import io.grpc.SynchronizationContext.ScheduledHandle; import io.grpc.internal.BackoffPolicy; import io.grpc.internal.TimeProvider; -import io.grpc.xds.XdsClientMetricReporter; -import io.grpc.xds.XdsClientMetricReporter.CallbackMetricReporter; import io.grpc.xds.client.Bootstrapper.AuthorityInfo; import io.grpc.xds.client.Bootstrapper.ServerInfo; import io.grpc.xds.client.XdsClient.ResourceMetadata.ResourceMetadataStatus; import io.grpc.xds.client.XdsClient.ResourceStore; +import io.grpc.xds.client.XdsClientMetricReporter; +import io.grpc.xds.client.XdsClientMetricReporter.CallbackMetricReporter; import io.grpc.xds.client.XdsLogger.XdsLogLevel; import java.net.URI; import java.util.Collection; @@ -127,7 +127,6 @@ public XdsClientImpl( this.securityConfig = securityConfig; this.target = target; this.metricReporter = metricReporter; - metricReporter.setXdsClient(this); logId = InternalLogId.allocate("xds-client", null); logger = XdsLogger.withLogId(logId); logger.log(XdsLogLevel.INFO, "Created"); @@ -191,7 +190,6 @@ public void run() { for (final LoadReportClient lrsClient : serverLrsClientMap.values()) { lrsClient.stopLoadReporting(); } - metricReporter.close(); cleanUpResourceTimers(); } }); diff --git a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporter.java b/xds/src/main/java/io/grpc/xds/client/XdsClientMetricReporter.java similarity index 68% rename from xds/src/main/java/io/grpc/xds/XdsClientMetricReporter.java rename to xds/src/main/java/io/grpc/xds/client/XdsClientMetricReporter.java index de5fafcf058..c64e0b7e42b 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporter.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClientMetricReporter.java @@ -14,10 +14,9 @@ * limitations under the License. */ -package io.grpc.xds; +package io.grpc.xds.client; import io.grpc.Internal; -import io.grpc.xds.client.XdsClient; /** * Interface for reporting metrics from the xDS client. @@ -48,45 +47,19 @@ default void reportResourceUpdates(long validResourceCount, long invalidResource default void reportServerFailure(long serverFailure, String target, String xdsServer) { } - /** - * Sets the {@link XdsClient} instance. - */ - default void setXdsClient(XdsClient xdsClient) { - } - - /** - * Closes the metric reporter. - */ - default void close() { - } - /** * Interface for reporting metrics through callback. * */ interface CallbackMetricReporter { - /** - * Reports number of resources in each cache state. - * - * @param resourceCount Number of resources. - * @param cacheState Status of the resource metadata - * {@link io.grpc.xds.client.XdsClient.ResourceMetadata.ResourceMetadataStatus}. - * @param resourceType Type of XDS resource (e.g., "envoy.config.listener.v3.Listener"). - * @param target Target of the gRPC channel. - */ + // TODO(@dnvindhya): include the "authority" label once xds.authority is available. default void reportResourceCounts(long resourceCount, String cacheState, String resourceType, String target) { } - /** - * Reports whether xDS client has a working ADS stream to the xDS server. - * - * @param isConnected 1 if the client is connected to the xDS server, 0 otherwise. - * @param target Target of the gRPC channel. - * @param xdsServer Target URI of the xDS server with which the XdsClient is communicating. - */ + default void reportServerConnections(int isConnected, String target, String xdsServer) { } } diff --git a/xds/src/test/java/io/grpc/xds/CsdsServiceTest.java b/xds/src/test/java/io/grpc/xds/CsdsServiceTest.java index 8002944eff2..7c6821dc560 100644 --- a/xds/src/test/java/io/grpc/xds/CsdsServiceTest.java +++ b/xds/src/test/java/io/grpc/xds/CsdsServiceTest.java @@ -554,10 +554,6 @@ public void setBootstrapOverride(Map bootstrap) { throw new UnsupportedOperationException("Should not be called"); } - @Override - public ObjectPool getOrCreate(String target) { - throw new UnsupportedOperationException("Should not be called"); - } @Override public ObjectPool getOrCreate(String target, MetricRecorder metricRecorder) { diff --git a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java index 4f59f715b46..2debd133cd3 100644 --- a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java +++ b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java @@ -96,6 +96,7 @@ import io.grpc.xds.client.XdsClient.ResourceUpdate; import io.grpc.xds.client.XdsClient.ResourceWatcher; import io.grpc.xds.client.XdsClientImpl; +import io.grpc.xds.client.XdsClientMetricReporter; import io.grpc.xds.client.XdsResourceType; import io.grpc.xds.client.XdsResourceType.ResourceInvalidException; import io.grpc.xds.client.XdsTransportFactory; diff --git a/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java b/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java index b1d3db2698e..4fb77f0be42 100644 --- a/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java @@ -66,9 +66,8 @@ public void noServer() throws XdsInitializationException { SharedXdsClientPoolProvider provider = new SharedXdsClientPoolProvider(bootstrapper); thrown.expect(XdsInitializationException.class); thrown.expectMessage("No xDS server provided"); - provider.getOrCreate(DUMMY_TARGET); + provider.getOrCreate(DUMMY_TARGET, metricRecorder); assertThat(provider.get(DUMMY_TARGET)).isNull(); - assertThat(provider.getMetricRecorder(DUMMY_TARGET)).isNull(); } @Test @@ -80,15 +79,11 @@ public void sharedXdsClientObjectPool() throws XdsInitializationException { SharedXdsClientPoolProvider provider = new SharedXdsClientPoolProvider(bootstrapper); assertThat(provider.get(DUMMY_TARGET)).isNull(); - assertThat(provider.getMetricRecorder(DUMMY_TARGET)).isNull(); - ObjectPool xdsClientPool = provider.getOrCreate(DUMMY_TARGET); - MetricRecorder metricRecorder = provider.getMetricRecorder(DUMMY_TARGET); + ObjectPool xdsClientPool = provider.getOrCreate(DUMMY_TARGET, metricRecorder); verify(bootstrapper).bootstrap(); - assertThat(provider.getOrCreate(DUMMY_TARGET)).isSameInstanceAs(xdsClientPool); + assertThat(provider.getOrCreate(DUMMY_TARGET, metricRecorder)).isSameInstanceAs(xdsClientPool); assertThat(provider.get(DUMMY_TARGET)).isNotNull(); assertThat(provider.get(DUMMY_TARGET)).isSameInstanceAs(xdsClientPool); - assertThat(provider.getMetricRecorder(DUMMY_TARGET)).isNotNull(); - assertThat(provider.getMetricRecorder(DUMMY_TARGET)).isSameInstanceAs(metricRecorder); verifyNoMoreInteractions(bootstrapper); } @@ -97,8 +92,9 @@ public void refCountedXdsClientObjectPool_delayedCreation() { ServerInfo server = ServerInfo.create(SERVER_URI, InsecureChannelCredentials.create()); BootstrapInfo bootstrapInfo = BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build(); + SharedXdsClientPoolProvider provider = new SharedXdsClientPoolProvider(bootstrapper); RefCountedXdsClientObjectPool xdsClientPool = - new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET, metricRecorder); + provider.new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET, metricRecorder); assertThat(xdsClientPool.getXdsClientForTest()).isNull(); XdsClient xdsClient = xdsClientPool.getObject(); assertThat(xdsClientPool.getXdsClientForTest()).isNotNull(); @@ -110,8 +106,9 @@ public void refCountedXdsClientObjectPool_refCounted() { ServerInfo server = ServerInfo.create(SERVER_URI, InsecureChannelCredentials.create()); BootstrapInfo bootstrapInfo = BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build(); + SharedXdsClientPoolProvider provider = new SharedXdsClientPoolProvider(bootstrapper); RefCountedXdsClientObjectPool xdsClientPool = - new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET, metricRecorder); + provider.new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET, metricRecorder); // getObject once XdsClient xdsClient = xdsClientPool.getObject(); assertThat(xdsClient).isNotNull(); @@ -130,8 +127,9 @@ public void refCountedXdsClientObjectPool_getObjectCreatesNewInstanceIfAlreadySh ServerInfo server = ServerInfo.create(SERVER_URI, InsecureChannelCredentials.create()); BootstrapInfo bootstrapInfo = BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build(); + SharedXdsClientPoolProvider provider = new SharedXdsClientPoolProvider(bootstrapper); RefCountedXdsClientObjectPool xdsClientPool = - new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET, metricRecorder); + provider.new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET, metricRecorder); XdsClient xdsClient1 = xdsClientPool.getObject(); assertThat(xdsClientPool.returnObject(xdsClient1)).isNull(); assertThat(xdsClient1.isShutDown()).isTrue(); diff --git a/xds/src/test/java/io/grpc/xds/XdsClientFederationTest.java b/xds/src/test/java/io/grpc/xds/XdsClientFederationTest.java index 0b8e89de721..b2b713e9a8e 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientFederationTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientFederationTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.grpc.MetricRecorder; import io.grpc.internal.ObjectPool; import io.grpc.xds.Filter.NamedFilterConfig; import io.grpc.xds.XdsListenerResource.LdsUpdate; @@ -73,12 +74,13 @@ public class XdsClientFederationTest { private ObjectPool xdsClientPool; private XdsClient xdsClient; private static final String DUMMY_TARGET = "dummy"; + private final MetricRecorder metricRecorder = new MetricRecorder() {}; @Before public void setUp() throws XdsInitializationException { SharedXdsClientPoolProvider clientPoolProvider = new SharedXdsClientPoolProvider(); clientPoolProvider.setBootstrapOverride(defaultBootstrapOverride()); - xdsClientPool = clientPoolProvider.getOrCreate(DUMMY_TARGET); + xdsClientPool = clientPoolProvider.getOrCreate(DUMMY_TARGET, metricRecorder); xdsClient = xdsClientPool.getObject(); } diff --git a/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java b/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java index 05c043955fa..b52b092c3e3 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -33,9 +34,9 @@ import io.grpc.MetricRecorder.BatchCallback; import io.grpc.MetricRecorder.BatchRecorder; import io.grpc.MetricSink; -import io.grpc.xds.XdsClientMetricReporter.CallbackMetricReporter; import io.grpc.xds.XdsClientMetricReporterImpl.CallbackMetricReporterImpl; import io.grpc.xds.client.XdsClient; +import io.grpc.xds.client.XdsClientMetricReporter.CallbackMetricReporter; import java.util.Arrays; import java.util.Collections; import org.junit.Before; @@ -139,16 +140,19 @@ public void setXdsClient_reportCallbackMetrics_resourceCountsFails() { @Test public void metricGauges() { + XdsClientMetricReporterImpl spyReporter = spy( + new XdsClientMetricReporterImpl(mockMetricRecorder)); SettableFuture future = SettableFuture.create(); future.set(null); when(mockXdsClient.reportResourceCounts(any(CallbackMetricReporter.class))) .thenReturn(future); when(mockXdsClient.reportServerConnections(any(CallbackMetricReporter.class))) .thenReturn(future); - CallbackMetricReporterImpl callbackMetricReporter = new CallbackMetricReporterImpl( + CallbackMetricReporter callbackMetricReporter = new CallbackMetricReporterImpl( mockBatchRecorder); - reporter.injectCallbackMetricReporter(callbackMetricReporter); - reporter.setXdsClient(mockXdsClient); + when(spyReporter.createCallbackMetricReporter(mockBatchRecorder)).thenReturn( + callbackMetricReporter); + spyReporter.setXdsClient(mockXdsClient); verify(mockMetricRecorder).registerBatchCallback(gaugeBatchCallbackCaptor.capture(), eqMetricInstrumentName("grpc.xds_client.connected"), eqMetricInstrumentName("grpc.xds_client.resources")); diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java index 71dd8fadccf..6f4c1503cee 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java @@ -216,11 +216,6 @@ public ObjectPool get(String target) { throw new UnsupportedOperationException("Should not be called"); } - @Override - public ObjectPool getOrCreate(String target) throws XdsInitializationException { - throw new XdsInitializationException("Fail to read bootstrap file"); - } - @Override public ObjectPool getOrCreate(String target, MetricRecorder metricRecorder) throws XdsInitializationException { @@ -2012,11 +2007,6 @@ public ObjectPool get(String target) { throw new UnsupportedOperationException("Should not be called"); } - @Override - public ObjectPool getOrCreate(String target) throws XdsInitializationException { - return getOrCreate(target, metricRecorder); - } - @Override public ObjectPool getOrCreate(String target, MetricRecorder metricRecorder) throws XdsInitializationException { diff --git a/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java b/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java index 6d80fc79bce..a27c2917712 100644 --- a/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java +++ b/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java @@ -151,11 +151,6 @@ public ObjectPool get(String target) { throw new UnsupportedOperationException("Should not be called"); } - @Override - public ObjectPool getOrCreate(String target) throws XdsInitializationException { - return getOrCreate(target, new MetricRecorder() {}); - } - @Override public ObjectPool getOrCreate(String target, MetricRecorder metricRecorder) throws XdsInitializationException { From 5881cb6676a8840bcd5254a602253c1bb9608a4a Mon Sep 17 00:00:00 2001 From: Vindhya Ningegowda Date: Mon, 11 Nov 2024 17:27:36 -0800 Subject: [PATCH 05/13] Move interfaces for reporting gauge values to XdsClient and update unit tests --- .../grpc/xds/XdsClientMetricReporterImpl.java | 44 ++++----- .../java/io/grpc/xds/client/XdsClient.java | 23 +++-- .../io/grpc/xds/client/XdsClientImpl.java | 30 ++---- .../xds/client/XdsClientMetricReporter.java | 16 ---- .../grpc/xds/GrpcXdsClientImplTestBase.java | 45 ++------- .../xds/XdsClientMetricReporterImplTest.java | 95 +++++++++++++------ 6 files changed, 119 insertions(+), 134 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java index 535efacd3d3..b86e7ebd85a 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java @@ -27,6 +27,8 @@ import io.grpc.MetricRecorder.BatchRecorder; import io.grpc.MetricRecorder.Registration; import io.grpc.xds.client.XdsClient; +import io.grpc.xds.client.XdsClient.ResourceCallback; +import io.grpc.xds.client.XdsClient.ServerConnectionCallback; import io.grpc.xds.client.XdsClientMetricReporter; import java.util.Arrays; import java.util.Collections; @@ -125,15 +127,15 @@ void close() { } void reportCallbackMetrics(BatchRecorder recorder) { - CallbackMetricReporter callbackMetricReporter = createCallbackMetricReporter(recorder); + MetricReporterCallback callback = new MetricReporterCallback(recorder); try { - SettableFuture ret = this.xdsClient.reportResourceCounts( - callbackMetricReporter); + SettableFuture reportResourceCountsCompleted = this.xdsClient.reportResourceCounts( + callback); + SettableFuture reportServerConnectionsCompleted = + this.xdsClient.reportServerConnections(callback); // Normally this shouldn't take long, but adding a timeout to avoid indefinite blocking - Void unused = ret.get(5, TimeUnit.SECONDS); - SettableFuture ret1 = this.xdsClient.reportServerConnections(callbackMetricReporter); - // Normally this shouldn't take long, but adding a timeout to avoid indefinite blocking - Void unused1 = ret1.get(5, TimeUnit.SECONDS); + Void unused1 = reportResourceCountsCompleted.get(5, TimeUnit.SECONDS); + Void unused2 = reportServerConnectionsCompleted.get(5, TimeUnit.SECONDS); } catch (Exception e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); // re-set the current thread's interruption state @@ -142,35 +144,27 @@ void reportCallbackMetrics(BatchRecorder recorder) { } } - /** - * Allows injecting a custom {@link CallbackMetricReporter} for testing purposes. - */ - @VisibleForTesting - CallbackMetricReporter createCallbackMetricReporter(BatchRecorder recorder) { - return new CallbackMetricReporterImpl(recorder); - } - @VisibleForTesting - static final class CallbackMetricReporterImpl implements - XdsClientMetricReporter.CallbackMetricReporter { + static final class MetricReporterCallback implements ResourceCallback, + ServerConnectionCallback { private final BatchRecorder recorder; - CallbackMetricReporterImpl(BatchRecorder recorder) { + MetricReporterCallback(BatchRecorder recorder) { this.recorder = recorder; } - @Override - public void reportServerConnections(int isConnected, String target, String xdsServer) { - recorder.recordLongGauge(CONNECTED_GAUGE, isConnected, Arrays.asList(target, xdsServer), - Collections.emptyList()); - } - // TODO(@dnvindhya): include the "authority" label once xds.authority is available. @Override - public void reportResourceCounts(long resourceCount, String cacheState, String resourceType, + public void reportResourceCountGauge(long resourceCount, String cacheState, String resourceType, String target) { recorder.recordLongGauge(RESOURCES_GAUGE, resourceCount, Arrays.asList(target, cacheState, resourceType), Collections.emptyList()); } + + @Override + public void reportServerConnectionGauge(int isConnected, String target, String xdsServer) { + recorder.recordLongGauge(CONNECTED_GAUGE, isConnected, Arrays.asList(target, xdsServer), + Collections.emptyList()); + } } } diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClient.java b/xds/src/main/java/io/grpc/xds/client/XdsClient.java index 428778fb216..74d32f0f94b 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClient.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClient.java @@ -29,7 +29,6 @@ import io.grpc.ExperimentalApi; import io.grpc.Status; import io.grpc.xds.client.Bootstrapper.ServerInfo; -import io.grpc.xds.client.XdsClientMetricReporter.CallbackMetricReporter; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -380,8 +379,20 @@ public Map getServerLrsClientMap() { throw new UnsupportedOperationException(); } + /** Callback used to report gauge metric value for resources. */ + public interface ResourceCallback { + // TODO(@dnvindhya): include the "authority" label once xds.authority is available. + void reportResourceCountGauge(long resourceCount, String cacheState, String resourceType, + String target); + } + + /** Callback used to report a gauge metric value for server connections. */ + public interface ServerConnectionCallback { + void reportServerConnectionGauge(int isConnected, String target, String xdsServer); + } + /** - * Reports the number of resources in each cache state through {@link CallbackMetricReporter}. + * Reports the number of resources in each cache state. * *

Cache state is determined by two factors: *

    @@ -390,16 +401,14 @@ public Map getServerLrsClientMap() { * resource. *
*/ - public SettableFuture reportResourceCounts(CallbackMetricReporter callbackMetricReporter) { + public SettableFuture reportResourceCounts(ResourceCallback callback) { throw new UnsupportedOperationException(); } /** - * Reports whether xDS client has a working ADS stream to xDS server. Reporting is done through - * {@link CallbackMetricReporter}. + * Reports whether xDS client has a working ADS stream to xDS server. */ - public SettableFuture reportServerConnections( - CallbackMetricReporter callbackMetricReporter) { + public SettableFuture reportServerConnections(ServerConnectionCallback callback) { throw new UnsupportedOperationException(); } diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java index f4244ff8354..b14c9c1befe 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java @@ -42,8 +42,6 @@ import io.grpc.xds.client.Bootstrapper.ServerInfo; import io.grpc.xds.client.XdsClient.ResourceMetadata.ResourceMetadataStatus; import io.grpc.xds.client.XdsClient.ResourceStore; -import io.grpc.xds.client.XdsClientMetricReporter; -import io.grpc.xds.client.XdsClientMetricReporter.CallbackMetricReporter; import io.grpc.xds.client.XdsLogger.XdsLogLevel; import java.net.URI; import java.util.Collection; @@ -536,12 +534,11 @@ private void handleResourceUpdate( } @Override - public SettableFuture reportServerConnections( - CallbackMetricReporter callbackMetricReporter) { + public SettableFuture reportServerConnections(ServerConnectionCallback callback) { SettableFuture future = SettableFuture.create(); syncContext.execute(() -> { serverCpClientMap.forEach((serverInfo, controlPlaneClient) -> - callbackMetricReporter.reportServerConnections( + callback.reportServerConnectionGauge( controlPlaneClient.hasWorkingAdsStream() ? 1 : 0, target, serverInfo.target())); @@ -551,12 +548,12 @@ public SettableFuture reportServerConnections( } @Override - public SettableFuture reportResourceCounts(CallbackMetricReporter callbackMetricReporter) { + public SettableFuture reportResourceCounts(ResourceCallback callback) { SettableFuture future = SettableFuture.create(); syncContext.execute(() -> { Map, Map> resourceCountsByType = getResourceCountsByType(); - reportResourceCountsToCallback(callbackMetricReporter, resourceCountsByType); + reportResourceCountsToCallback(callback, resourceCountsByType); }); future.set(null); return future; @@ -568,9 +565,7 @@ private String cacheStateFromResourceStatus(ResourceMetadata metadata, boolean i ? status + "_but_cached" : status; } - /** - * Calculates number of resources by ResourceType and ResourceSubscriber.metadata.status. - */ + /** Calculates number of resources by ResourceType and ResourceSubscriber.metadata.status. */ Map, Map> getResourceCountsByType() { Map, Map> resourceCountsByType = new HashMap<>(); for (XdsResourceType resourceType : resourceSubscribers.keySet()) { @@ -586,10 +581,8 @@ Map, Map> getResourceCountsByType() { return resourceCountsByType; } - /** - * Reports resource counts using the provided callbackMetricReporter. - */ - void reportResourceCountsToCallback(CallbackMetricReporter callbackMetricReporter, + /** Reports resource counts using the provided ResourceCallback. */ + void reportResourceCountsToCallback(ResourceCallback callback, Map, Map> resourceCountsByType) { for (Map.Entry, Map> entry : resourceCountsByType.entrySet()) { @@ -597,17 +590,12 @@ void reportResourceCountsToCallback(CallbackMetricReporter callbackMetricReporte Map resourceCountsByState = entry.getValue(); // TODO(@dnvindhya): include the "authority" label once authority is available here. resourceCountsByState.forEach((cacheState, count) -> - callbackMetricReporter.reportResourceCounts( - count, - cacheState, resourceType.typeUrl(), target - )); + callback.reportResourceCountGauge(count, cacheState, resourceType.typeUrl(), target)); } } - /** - * Tracks a single subscribed resource. - */ + /** Tracks a single subscribed resource. */ private final class ResourceSubscriber { @Nullable private final ServerInfo serverInfo; @Nullable private final ControlPlaneClient controlPlaneClient; diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClientMetricReporter.java b/xds/src/main/java/io/grpc/xds/client/XdsClientMetricReporter.java index c64e0b7e42b..1657c1aae20 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClientMetricReporter.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClientMetricReporter.java @@ -47,20 +47,4 @@ default void reportResourceUpdates(long validResourceCount, long invalidResource default void reportServerFailure(long serverFailure, String target, String xdsServer) { } - /** - * Interface for reporting metrics through callback. - * - */ - interface CallbackMetricReporter { - - - // TODO(@dnvindhya): include the "authority" label once xds.authority is available. - default void reportResourceCounts(long resourceCount, String cacheState, String resourceType, - String target) { - } - - - default void reportServerConnections(int isConnected, String target, String xdsServer) { - } - } } diff --git a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java index 2debd133cd3..9bf14524559 100644 --- a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java +++ b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java @@ -90,11 +90,13 @@ import io.grpc.xds.client.EnvoyProtoData.Node; import io.grpc.xds.client.LoadStatsManager2.ClusterDropStats; import io.grpc.xds.client.Locality; +import io.grpc.xds.client.XdsClient.ResourceCallback; import io.grpc.xds.client.XdsClient.ResourceMetadata; import io.grpc.xds.client.XdsClient.ResourceMetadata.ResourceMetadataStatus; import io.grpc.xds.client.XdsClient.ResourceMetadata.UpdateFailureState; import io.grpc.xds.client.XdsClient.ResourceUpdate; import io.grpc.xds.client.XdsClient.ResourceWatcher; +import io.grpc.xds.client.XdsClient.ServerConnectionCallback; import io.grpc.xds.client.XdsClientImpl; import io.grpc.xds.client.XdsClientMetricReporter; import io.grpc.xds.client.XdsResourceType; @@ -294,7 +296,9 @@ public long currentTimeNanos() { @Mock private XdsClientMetricReporter xdsClientMetricReporter; @Mock - private XdsClientMetricReporter.CallbackMetricReporter callbackMetricReporter; + private ResourceCallback resourceCallback; + @Mock + private ServerConnectionCallback serverConnectionCallback; private ManagedChannel channel; private ManagedChannel channelForCustomAuthority; @@ -642,7 +646,7 @@ private void verifyServerFailureCount(int times, long serverFailureCount, String */ private void callback_ReportResourceCount() { try { - Future unused = xdsClient.reportResourceCounts(callbackMetricReporter); + Future unused = xdsClient.reportResourceCounts(resourceCallback); } catch (Exception e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); @@ -653,7 +657,7 @@ private void callback_ReportResourceCount() { private void verifyResourceCountByCacheState(int times, long resourceCount, String cacheState, String resourceTypeUrl, String dataPlaneTarget) { - verify(callbackMetricReporter, times(times)).reportResourceCounts( + verify(resourceCallback, times(times)).reportResourceCountGauge( eq(resourceCount), eq(cacheState), eq(resourceTypeUrl), @@ -666,7 +670,7 @@ private void verifyResourceCountByCacheState(int times, long resourceCount, */ private void callback_ReportServerConnection() { try { - Future unused = xdsClient.reportServerConnections(callbackMetricReporter); + Future unused = xdsClient.reportServerConnections(serverConnectionCallback); } catch (Exception e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); @@ -677,7 +681,7 @@ private void callback_ReportServerConnection() { private void verifyServerConnection(int times, int isConnected, String dataPlaneTarget, String xdsServer) { - verify(callbackMetricReporter, times(times)).reportServerConnections( + verify(serverConnectionCallback, times(times)).reportServerConnectionGauge( eq(isConnected), eq(dataPlaneTarget), eq(xdsServer)); @@ -709,7 +713,6 @@ public void ldsResourceNotFound() { verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); callback_ReportResourceCount(); verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl(), CHANNEL_TARGET); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -972,7 +975,6 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti // Verify {C.1} stays in the previous version VERSION_1, no matter {C} is deleted or not. verifyResourceMetadataAcked(RDS, "C.1", resourcesV11.get("C.1"), VERSION_1, TIME_INCREMENT * 2); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -1043,7 +1045,6 @@ public void ldsResourceFound_containsRdsName() { // Check metric data. callback_ReportResourceCount(); verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -1112,7 +1113,6 @@ public void ldsResourceUpdated() { verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); assertThat(channelForCustomAuthority).isNull(); assertThat(channelForEmptyAuthority).isNull(); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -1151,7 +1151,6 @@ public void cancelResourceWatcherNotRemoveUrlSubscribers() { // Check metric data. callback_ReportResourceCount(); verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -1233,7 +1232,6 @@ public void ldsResourceUpdated_withXdstpResourceName_withWrongType() { // Check metric data. callback_ReportResourceCount(); verifyResourceCountByCacheState(2, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -1416,7 +1414,6 @@ public void ldsResourceUpdate_withFaultInjection() { assertThat(faultConfig.faultAbort().percent().denominatorType()) .isEqualTo(DenominatorType.MILLION); assertThat(faultConfig.maxActiveFaults()).isEqualTo(101); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -1450,7 +1447,6 @@ public void ldsResourceDeleted() { // Check metric data. callback_ReportResourceCount(); verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl(), CHANNEL_TARGET); - verifyNoMoreInteractions(callbackMetricReporter); } /** @@ -1504,7 +1500,6 @@ public void ldsResourceDeleted_ignoreResourceDeletion() { callback_ReportResourceCount(); verifyResourceCountByCacheState(3, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); verifyNoMoreInteractions(ldsResourceWatcher); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -1557,7 +1552,6 @@ public void multipleLdsWatchers() { // Check metric data. callback_ReportResourceCount(); verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -1668,7 +1662,6 @@ public void rdsResponseErrorHandling_nackWeightedSumZero() { // The response is NACKed with the same error message. call.verifyRequestNack(RDS, RDS_RESOURCE, "", "0000", NODE, errors); verify(rdsResourceWatcher, never()).onChanged(any(RdsUpdate.class)); - verifyNoMoreInteractions(callbackMetricReporter); } /** @@ -1752,7 +1745,6 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { callback_ReportResourceCount(); verifyResourceCountByCacheState(2, 3, "acked", RDS.typeUrl(), CHANNEL_TARGET); verifySubscribedResourcesMetadataSizes(0, 0, 3, 0); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -1853,7 +1845,6 @@ public void rdsResourceUpdated() { // Check metric data. callback_ReportResourceCount(); verifyResourceCountByCacheState(2, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -1909,7 +1900,6 @@ public void rdsResourceDeletedByLdsApiListener() { callback_ReportResourceCount(); verifyResourceCountByCacheState(3, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); verifyResourceCountByCacheState(2, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -1993,7 +1983,6 @@ public void rdsResourcesDeletedByLdsTcpListener() { callback_ReportResourceCount(); verifyResourceCountByCacheState(3, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); verifyResourceCountByCacheState(2, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -2216,7 +2205,6 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataAcked(CDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); call.verifyRequest(CDS, subscribedResourceNames, VERSION_3, "0002", NODE); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -2330,7 +2318,6 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti // Verify {C.1} stays in the previous version VERSION_1. {C1} deleted or not does not matter. verifyResourceMetadataAcked(EDS, "C.1", resourcesV11.get("C.1"), VERSION_1, TIME_INCREMENT * 2); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -2587,7 +2574,6 @@ public void cdsResponseErrorHandling_badUpstreamTlsContext() { // Check metric data. callback_ReportResourceCount(); verifyResourceCountByCacheState(2, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); - verifyNoMoreInteractions(callbackMetricReporter); } /** @@ -2663,7 +2649,6 @@ public void cdsResponseWithOutlierDetection() { // Check metric data. callback_ReportResourceCount(); verifyResourceCountByCacheState(1, 1, "acked", CDS.typeUrl(), CHANNEL_TARGET); - verifyNoMoreInteractions(callbackMetricReporter); } /** @@ -3183,7 +3168,6 @@ public void edsCleanupNonceAfterUnsubscription() { // Check metric data. callback_ReportResourceCount(); verifyResourceCountByCacheState(2, 1, "requested", EDS.typeUrl(), CHANNEL_TARGET); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -3313,7 +3297,6 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { // Check metric data. callback_ReportResourceCount(); verifyResourceCountByCacheState(2, 3, "acked", EDS.typeUrl(), CHANNEL_TARGET); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -3453,7 +3436,6 @@ public void flowControlAbsent() throws Exception { callback_ReportResourceCount(); verifyResourceCountByCacheState(3, 1, "acked", CDS.typeUrl(), CHANNEL_TARGET); verifyResourceCountByCacheState(1, 1, "nacked", CDS.typeUrl(), CHANNEL_TARGET); - verifyNoMoreInteractions(callbackMetricReporter); } private Answer blockUpdate(CyclicBarrier barrier) { @@ -3691,7 +3673,6 @@ public void edsResourceDeletedByCds() { callback_ReportResourceCount(); verifyResourceCountByCacheState(3, 2, "acked", CDS.typeUrl(), CHANNEL_TARGET); verifyResourceCountByCacheState(2, 2, "acked", EDS.typeUrl(), CHANNEL_TARGET); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -3806,7 +3787,6 @@ public void streamClosedWithNoResponse() { verify(rdsResourceWatcher).onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, "ADS stream closed with OK before receiving a response"); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -3985,7 +3965,6 @@ public void streamClosedAndRetryWithBackoff() { verifyServerConnection(5, 0, CHANNEL_TARGET, xdsServerInfo.target()); inOrder.verifyNoMoreInteractions(); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -4039,7 +4018,7 @@ public void streamClosedAndRetryRaceWithAddRemoveWatchers() { callback_ReportServerConnection(); verifyServerConnection(2, 1, CHANNEL_TARGET, xdsServerInfo.target()); - verifyNoMoreInteractions(ldsResourceWatcher, rdsResourceWatcher, callbackMetricReporter); + verifyNoMoreInteractions(ldsResourceWatcher, rdsResourceWatcher); } @Test @@ -4109,7 +4088,6 @@ public void streamClosedAndRetryRestartsResourceInitialFetchTimerForUnresolvedRe assertThat(fakeClock.getPendingTasks(RDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(0); assertThat(fakeClock.getPendingTasks(CDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); assertThat(fakeClock.getPendingTasks(EDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(1); - verifyNoMoreInteractions(callbackMetricReporter); } @Test @@ -4320,8 +4298,6 @@ public void sendingToStoppedServer() throws Exception { callback_ReportResourceCount(); verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); verifyResourceCountByCacheState(4, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); - - verifyNoMoreInteractions(callbackMetricReporter); } catch (Throwable t) { throw t; // This allows putting a breakpoint here for debugging } @@ -4593,7 +4569,6 @@ public void testReportResourceCounts() { verifyResourceMetadataAcked(LDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); call.verifyRequest(LDS, subscribedResourceNames, VERSION_3, "0002", NODE); verifySubscribedResourcesMetadataSizes(3, 0, 0, 0); - verifyNoMoreInteractions(callbackMetricReporter); } private XdsClientImpl createXdsClient(String serverUri) { diff --git a/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java b/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java index b52b092c3e3..eb7264d6094 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java @@ -16,13 +16,12 @@ package io.grpc.xds; +import static com.google.common.truth.Truth.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -34,11 +33,18 @@ import io.grpc.MetricRecorder.BatchCallback; import io.grpc.MetricRecorder.BatchRecorder; import io.grpc.MetricSink; -import io.grpc.xds.XdsClientMetricReporterImpl.CallbackMetricReporterImpl; +import io.grpc.xds.XdsClientMetricReporterImpl.MetricReporterCallback; import io.grpc.xds.client.XdsClient; -import io.grpc.xds.client.XdsClientMetricReporter.CallbackMetricReporter; +import io.grpc.xds.client.XdsClient.ResourceCallback; +import io.grpc.xds.client.XdsClient.ServerConnectionCallback; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.List; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -110,66 +116,95 @@ public void reportServerFailure() { public void setXdsClient_reportMetrics() throws Exception { SettableFuture future = SettableFuture.create(); future.set(null); - when(mockXdsClient.reportResourceCounts(any(CallbackMetricReporter.class))) + when(mockXdsClient.reportResourceCounts(any(ResourceCallback.class))) .thenReturn(future); - when(mockXdsClient.reportServerConnections(any(CallbackMetricReporter.class))) + when(mockXdsClient.reportServerConnections(any(ServerConnectionCallback.class))) .thenReturn(future); reporter.setXdsClient(mockXdsClient); verify(mockMetricRecorder).registerBatchCallback(gaugeBatchCallbackCaptor.capture(), eqMetricInstrumentName("grpc.xds_client.connected"), eqMetricInstrumentName("grpc.xds_client.resources")); gaugeBatchCallbackCaptor.getValue().accept(mockBatchRecorder); - verify(mockXdsClient).reportResourceCounts(any(CallbackMetricReporter.class)); - verify(mockXdsClient).reportServerConnections(any(CallbackMetricReporter.class)); + verify(mockXdsClient).reportResourceCounts(any(ResourceCallback.class)); + verify(mockXdsClient).reportServerConnections(any(ServerConnectionCallback.class)); } @Test public void setXdsClient_reportCallbackMetrics_resourceCountsFails() { + List logs = new ArrayList<>(); + Handler testLogHandler = new Handler() { + @Override + public void publish(LogRecord record) { + logs.add(record); + } + + @Override + public void close() {} + + @Override + public void flush() {} + }; + Logger logger = Logger.getLogger(XdsClientMetricReporterImpl.class.getName()); + logger.addHandler(testLogHandler); + + // Create a future that will throw an exception SettableFuture resourceCountsFuture = SettableFuture.create(); resourceCountsFuture.setException(new Exception("test")); when(mockXdsClient.reportResourceCounts(any())).thenReturn(resourceCountsFuture); + // For server connections, return a normally completed future + SettableFuture serverConnectionsFuture = SettableFuture.create(); + serverConnectionsFuture.set(null); + when(mockXdsClient.reportServerConnections(any())).thenReturn(serverConnectionsFuture); + reporter.setXdsClient(mockXdsClient); verify(mockMetricRecorder) .registerBatchCallback(gaugeBatchCallbackCaptor.capture(), any(), any()); gaugeBatchCallbackCaptor.getValue().accept(mockBatchRecorder); + // Verify that the xdsClient methods were called + verify(mockXdsClient).reportResourceCounts(any(ResourceCallback.class)); + verify(mockXdsClient).reportServerConnections(any(ServerConnectionCallback.class)); - verify(mockXdsClient).reportResourceCounts(any(CallbackMetricReporter.class)); - verify(mockXdsClient, never()).reportServerConnections(any(CallbackMetricReporter.class)); + assertThat(logs.size()).isEqualTo(1); + assertThat(logs.get(0).getLevel()).isEqualTo(Level.WARNING); + assertThat(logs.get(0).getMessage()).isEqualTo("Failed to report gauge metrics"); + logger.removeHandler(testLogHandler); } @Test public void metricGauges() { - XdsClientMetricReporterImpl spyReporter = spy( - new XdsClientMetricReporterImpl(mockMetricRecorder)); SettableFuture future = SettableFuture.create(); future.set(null); - when(mockXdsClient.reportResourceCounts(any(CallbackMetricReporter.class))) + when(mockXdsClient.reportResourceCounts(any(ResourceCallback.class))) .thenReturn(future); - when(mockXdsClient.reportServerConnections(any(CallbackMetricReporter.class))) + when(mockXdsClient.reportServerConnections(any(ServerConnectionCallback.class))) .thenReturn(future); - CallbackMetricReporter callbackMetricReporter = new CallbackMetricReporterImpl( - mockBatchRecorder); - when(spyReporter.createCallbackMetricReporter(mockBatchRecorder)).thenReturn( - callbackMetricReporter); - spyReporter.setXdsClient(mockXdsClient); + reporter.setXdsClient(mockXdsClient); verify(mockMetricRecorder).registerBatchCallback(gaugeBatchCallbackCaptor.capture(), eqMetricInstrumentName("grpc.xds_client.connected"), eqMetricInstrumentName("grpc.xds_client.resources")); BatchCallback gaugeBatchCallback = gaugeBatchCallbackCaptor.getValue(); - // Verify the correct resource gauge values when requested at this point. InOrder inOrder = inOrder(mockBatchRecorder); + // Trigger the internal call to reportCallbackMetrics() gaugeBatchCallback.accept(mockBatchRecorder); - verify(mockXdsClient).reportResourceCounts(eq(callbackMetricReporter)); - verify(mockXdsClient).reportServerConnections(eq(callbackMetricReporter)); + ArgumentCaptor resourceCallbackCaptor = + ArgumentCaptor.forClass(ResourceCallback.class); + ArgumentCaptor serverConnectionCallbackCaptor = + ArgumentCaptor.forClass(ServerConnectionCallback.class); + verify(mockXdsClient).reportResourceCounts(resourceCallbackCaptor.capture()); + verify(mockXdsClient).reportServerConnections(serverConnectionCallbackCaptor.capture()); - callbackMetricReporter.reportResourceCounts(10, "acked", resourceTypeUrl, target); + // Get the captured callback + MetricReporterCallback callback = (MetricReporterCallback) resourceCallbackCaptor.getValue(); + + // Verify that reportResourceCounts and reportServerConnections were called + // with the captured callback + callback.reportResourceCountGauge(10, "acked", resourceTypeUrl, target); inOrder.verify(mockBatchRecorder) .recordLongGauge(eqMetricInstrumentName("grpc.xds_client.resources"), eq(10L), any(), any()); - - callbackMetricReporter.reportServerConnections(1, target, "xdsServer"); + callback.reportServerConnectionGauge(1, target, "xdsServer"); inOrder.verify(mockBatchRecorder) .recordLongGauge(eqMetricInstrumentName("grpc.xds_client.connected"), eq(1L), any(), any()); @@ -177,18 +212,18 @@ public void metricGauges() { } @Test - public void callbackMetricReporter() { - XdsClientMetricReporterImpl.CallbackMetricReporterImpl callback = - new XdsClientMetricReporterImpl.CallbackMetricReporterImpl(mockBatchRecorder); + public void metricReporterCallback() { + MetricReporterCallback callback = + new MetricReporterCallback(mockBatchRecorder); - callback.reportServerConnections(1, target, server); + callback.reportServerConnectionGauge(1, target, server); verify(mockBatchRecorder, times(1)).recordLongGauge( eqMetricInstrumentName("grpc.xds_client.connected"), eq(1L), eq(Lists.newArrayList(target, server)), eq(Lists.newArrayList())); String cacheState = "requested"; - callback.reportResourceCounts(10, cacheState, resourceTypeUrl, target); + callback.reportResourceCountGauge(10, cacheState, resourceTypeUrl, target); verify(mockBatchRecorder, times(1)).recordLongGauge( eqMetricInstrumentName("grpc.xds_client.resources"), eq(10L), eq(Arrays.asList(target, cacheState, resourceTypeUrl)), From 4831431c544f2ec25c6576832109a91422e27829 Mon Sep 17 00:00:00 2001 From: Vindhya Ningegowda Date: Tue, 12 Nov 2024 14:10:17 -0800 Subject: [PATCH 06/13] Updated `ServerConnectionCallback#reportServerConnectionGauge` method signature and addressed review comments. --- .../grpc/xds/XdsClientMetricReporterImpl.java | 6 +-- .../java/io/grpc/xds/client/XdsClient.java | 5 +- .../io/grpc/xds/client/XdsClientImpl.java | 8 ++- .../grpc/xds/GrpcXdsClientImplTestBase.java | 52 +++++++++---------- .../xds/XdsClientMetricReporterImplTest.java | 8 +-- 5 files changed, 40 insertions(+), 39 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java index b86e7ebd85a..6361a42bc53 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java @@ -162,9 +162,9 @@ public void reportResourceCountGauge(long resourceCount, String cacheState, Stri } @Override - public void reportServerConnectionGauge(int isConnected, String target, String xdsServer) { - recorder.recordLongGauge(CONNECTED_GAUGE, isConnected, Arrays.asList(target, xdsServer), - Collections.emptyList()); + public void reportServerConnectionGauge(boolean isConnected, String target, String xdsServer) { + recorder.recordLongGauge(CONNECTED_GAUGE, isConnected ? 1 : 0, + Arrays.asList(target, xdsServer), Collections.emptyList()); } } } diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClient.java b/xds/src/main/java/io/grpc/xds/client/XdsClient.java index 74d32f0f94b..424d3b99e3d 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClient.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClient.java @@ -388,7 +388,7 @@ void reportResourceCountGauge(long resourceCount, String cacheState, String reso /** Callback used to report a gauge metric value for server connections. */ public interface ServerConnectionCallback { - void reportServerConnectionGauge(int isConnected, String target, String xdsServer); + void reportServerConnectionGauge(boolean isConnected, String target, String xdsServer); } /** @@ -407,6 +407,9 @@ public SettableFuture reportResourceCounts(ResourceCallback callback) { /** * Reports whether xDS client has a working ADS stream to xDS server. + * The definition of a working stream is defined in gRFC A78. + * @see + * A78-grpc-metrics-wrr-pf-xds.md */ public SettableFuture reportServerConnections(ServerConnectionCallback callback) { throw new UnsupportedOperationException(); diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java index b14c9c1befe..7bdf960b9b9 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java @@ -539,11 +539,9 @@ public SettableFuture reportServerConnections(ServerConnectionCallback cal syncContext.execute(() -> { serverCpClientMap.forEach((serverInfo, controlPlaneClient) -> callback.reportServerConnectionGauge( - controlPlaneClient.hasWorkingAdsStream() ? 1 : 0, - target, - serverInfo.target())); + controlPlaneClient.hasWorkingAdsStream(), target, serverInfo.target())); + future.set(null); }); - future.set(null); return future; } @@ -554,8 +552,8 @@ public SettableFuture reportResourceCounts(ResourceCallback callback) { Map, Map> resourceCountsByType = getResourceCountsByType(); reportResourceCountsToCallback(callback, resourceCountsByType); + future.set(null); }); - future.set(null); return future; } diff --git a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java index 9bf14524559..d08fb5b5558 100644 --- a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java +++ b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java @@ -679,7 +679,7 @@ private void callback_ReportServerConnection() { } } - private void verifyServerConnection(int times, int isConnected, String dataPlaneTarget, + private void verifyServerConnection(int times, boolean isConnected, String dataPlaneTarget, String xdsServer) { verify(serverConnectionCallback, times(times)).reportServerConnectionGauge( eq(isConnected), @@ -3774,12 +3774,12 @@ public void streamClosedWithNoResponse() { DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(1, 1, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(1, true, CHANNEL_TARGET, xdsServerInfo.target()); // Management server closes the RPC stream before sending any response. call.sendCompleted(); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(1, 0, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(1, false, CHANNEL_TARGET, xdsServerInfo.target()); verify(ldsResourceWatcher, Mockito.timeout(1000).times(1)) .onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, @@ -3797,7 +3797,7 @@ public void streamClosedAfterSendingResponses() { DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(1, 1, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(1, true, CHANNEL_TARGET, xdsServerInfo.target()); ScheduledTask ldsResourceTimeout = Iterables.getOnlyElement(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); ScheduledTask rdsResourceTimeout = @@ -3805,7 +3805,7 @@ public void streamClosedAfterSendingResponses() { call.sendResponse(LDS, testListenerRds, VERSION_1, "0000"); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(2, 1, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(2, true, CHANNEL_TARGET, xdsServerInfo.target()); assertThat(ldsResourceTimeout.isCancelled()).isTrue(); call.sendResponse(RDS, testRouteConfig, VERSION_1, "0000"); assertThat(rdsResourceTimeout.isCancelled()).isTrue(); @@ -3813,7 +3813,7 @@ public void streamClosedAfterSendingResponses() { call.sendCompleted(); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(3, 1, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(3, true, CHANNEL_TARGET, xdsServerInfo.target()); verify(ldsResourceWatcher, never()).onError(errorCaptor.capture()); verify(rdsResourceWatcher, never()).onError(errorCaptor.capture()); } @@ -3824,7 +3824,7 @@ public void streamClosedAndRetryWithBackoff() { xdsClient.watchXdsResource(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(1, 1, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(1, true, CHANNEL_TARGET, xdsServerInfo.target()); xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, rdsResourceWatcher); xdsClient.watchXdsResource(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); @@ -3849,7 +3849,7 @@ public void streamClosedAndRetryWithBackoff() { // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(1, 0, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(1, false, CHANNEL_TARGET, xdsServerInfo.target()); // Retry after backoff. inOrder.verify(backoffPolicyProvider).get(); @@ -3866,7 +3866,7 @@ public void streamClosedAndRetryWithBackoff() { // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(2, 0, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(2, false, CHANNEL_TARGET, xdsServerInfo.target()); // Management server becomes unreachable. String errorMsg = "my fault"; @@ -3882,7 +3882,7 @@ public void streamClosedAndRetryWithBackoff() { // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(3, 0, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(3, false, CHANNEL_TARGET, xdsServerInfo.target()); // Retry after backoff. inOrder.verify(backoffPolicy1).nextBackoffNanos(); @@ -3903,7 +3903,7 @@ public void streamClosedAndRetryWithBackoff() { call.verifyRequest(LDS, LDS_RESOURCE, "63", "3242", NODE); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(2, 1, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(2, true, CHANNEL_TARGET, xdsServerInfo.target()); List routeConfigs = ImmutableList.of( Any.pack(mf.buildRouteConfiguration(RDS_RESOURCE, mf.buildOpaqueVirtualHosts(2)))); @@ -3920,7 +3920,7 @@ public void streamClosedAndRetryWithBackoff() { // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(3, 1, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(3, true, CHANNEL_TARGET, xdsServerInfo.target()); // Reset backoff sequence and retry after backoff. inOrder.verify(backoffPolicyProvider).get(); @@ -3946,7 +3946,7 @@ public void streamClosedAndRetryWithBackoff() { // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(4, 0, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(4, false, CHANNEL_TARGET, xdsServerInfo.target()); // Retry after backoff. inOrder.verify(backoffPolicy2).nextBackoffNanos(); @@ -3962,7 +3962,7 @@ public void streamClosedAndRetryWithBackoff() { // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(5, 0, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(5, false, CHANNEL_TARGET, xdsServerInfo.target()); inOrder.verifyNoMoreInteractions(); } @@ -3976,7 +3976,7 @@ public void streamClosedAndRetryRaceWithAddRemoveWatchers() { DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(1, 1, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(1, true, CHANNEL_TARGET, xdsServerInfo.target()); call.sendError(Status.UNAVAILABLE.asException()); verify(ldsResourceWatcher, Mockito.timeout(1000).times(1)) .onError(errorCaptor.capture()); @@ -3989,7 +3989,7 @@ public void streamClosedAndRetryRaceWithAddRemoveWatchers() { // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(1, 0, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(1, false, CHANNEL_TARGET, xdsServerInfo.target()); xdsClient.cancelXdsResourceWatch(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); @@ -4007,7 +4007,7 @@ public void streamClosedAndRetryRaceWithAddRemoveWatchers() { // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(2,0, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(2,false, CHANNEL_TARGET, xdsServerInfo.target()); call.sendResponse(LDS, testListenerRds, VERSION_1, "0000"); List routeConfigs = ImmutableList.of( @@ -4016,7 +4016,7 @@ public void streamClosedAndRetryRaceWithAddRemoveWatchers() { // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(2, 1, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(2, true, CHANNEL_TARGET, xdsServerInfo.target()); verifyNoMoreInteractions(ldsResourceWatcher, rdsResourceWatcher); } @@ -4031,7 +4031,7 @@ public void streamClosedAndRetryRestartsResourceInitialFetchTimerForUnresolvedRe DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(1, 1, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(1, true, CHANNEL_TARGET, xdsServerInfo.target()); callback_ReportResourceCount(); verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl(), CHANNEL_TARGET); @@ -4049,7 +4049,7 @@ public void streamClosedAndRetryRestartsResourceInitialFetchTimerForUnresolvedRe assertThat(ldsResourceTimeout.isCancelled()).isTrue(); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(2, 1, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(2, true, CHANNEL_TARGET, xdsServerInfo.target()); callback_ReportResourceCount(); verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); verifyResourceCountByCacheState(2, 1, "requested", RDS.typeUrl(), CHANNEL_TARGET); @@ -4060,7 +4060,7 @@ public void streamClosedAndRetryRestartsResourceInitialFetchTimerForUnresolvedRe assertThat(rdsResourceTimeout.isCancelled()).isTrue(); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(3, 1, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(3, true, CHANNEL_TARGET, xdsServerInfo.target()); callback_ReportResourceCount(); verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); verifyResourceCountByCacheState(1, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); @@ -4076,7 +4076,7 @@ public void streamClosedAndRetryRestartsResourceInitialFetchTimerForUnresolvedRe verify(edsResourceWatcher, never()).onError(errorCaptor.capture()); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(4, 1, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(4, true, CHANNEL_TARGET, xdsServerInfo.target()); callback_ReportResourceCount(); verifyResourceCountByCacheState(3, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); verifyResourceCountByCacheState(2, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); @@ -4245,7 +4245,7 @@ public void sendingToStoppedServer() throws Exception { fakeClock.forwardTime(14, TimeUnit.SECONDS); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(1, 0, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(1, false, CHANNEL_TARGET, xdsServerInfo.target()); callback_ReportResourceCount(); verifyResourceCountByCacheState(1, 1, "unknown", LDS.typeUrl(), CHANNEL_TARGET); verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); @@ -4265,7 +4265,7 @@ public void sendingToStoppedServer() throws Exception { DiscoveryRpcCall call = resourceDiscoveryCalls.poll(3, TimeUnit.SECONDS); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(2, 0, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(2, false, CHANNEL_TARGET, xdsServerInfo.target()); callback_ReportResourceCount(); verifyResourceCountByCacheState(2, 1, "unknown", LDS.typeUrl(), CHANNEL_TARGET); verifyResourceCountByCacheState(2, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); @@ -4276,7 +4276,7 @@ public void sendingToStoppedServer() throws Exception { // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(3, 0, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(3, false, CHANNEL_TARGET, xdsServerInfo.target()); callback_ReportResourceCount(); verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); verifyResourceCountByCacheState(3, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); @@ -4294,7 +4294,7 @@ public void sendingToStoppedServer() throws Exception { verifySubscribedResourcesMetadataSizes(1, 1, 0, 0); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(1, 1, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(1, true, CHANNEL_TARGET, xdsServerInfo.target()); callback_ReportResourceCount(); verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); verifyResourceCountByCacheState(4, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); diff --git a/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java b/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java index eb7264d6094..b69e7c7c27a 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java @@ -162,8 +162,8 @@ public void flush() {} .registerBatchCallback(gaugeBatchCallbackCaptor.capture(), any(), any()); gaugeBatchCallbackCaptor.getValue().accept(mockBatchRecorder); // Verify that the xdsClient methods were called - verify(mockXdsClient).reportResourceCounts(any(ResourceCallback.class)); - verify(mockXdsClient).reportServerConnections(any(ServerConnectionCallback.class)); + verify(mockXdsClient).reportResourceCounts(any()); + verify(mockXdsClient).reportServerConnections(any()); assertThat(logs.size()).isEqualTo(1); assertThat(logs.get(0).getLevel()).isEqualTo(Level.WARNING); @@ -204,7 +204,7 @@ public void metricGauges() { inOrder.verify(mockBatchRecorder) .recordLongGauge(eqMetricInstrumentName("grpc.xds_client.resources"), eq(10L), any(), any()); - callback.reportServerConnectionGauge(1, target, "xdsServer"); + callback.reportServerConnectionGauge(true, target, "xdsServer"); inOrder.verify(mockBatchRecorder) .recordLongGauge(eqMetricInstrumentName("grpc.xds_client.connected"), eq(1L), any(), any()); @@ -216,7 +216,7 @@ public void metricReporterCallback() { MetricReporterCallback callback = new MetricReporterCallback(mockBatchRecorder); - callback.reportServerConnectionGauge(1, target, server); + callback.reportServerConnectionGauge(true, target, server); verify(mockBatchRecorder, times(1)).recordLongGauge( eqMetricInstrumentName("grpc.xds_client.connected"), eq(1L), eq(Lists.newArrayList(target, server)), From cbe2659550224e041fbde45a3d35774eaf2fe2c6 Mon Sep 17 00:00:00 2001 From: Vindhya Ningegowda Date: Thu, 14 Nov 2024 10:31:50 -0800 Subject: [PATCH 07/13] Inject value from XdsClientMetricReporterImpl and addressed review comments --- .../grpc/xds/SharedXdsClientPoolProvider.java | 9 +- .../grpc/xds/XdsClientMetricReporterImpl.java | 22 +- .../java/io/grpc/xds/XdsNameResolver.java | 2 +- .../java/io/grpc/xds/XdsServerWrapper.java | 2 + .../java/io/grpc/xds/client/XdsClient.java | 7 +- .../io/grpc/xds/client/XdsClientImpl.java | 53 +- .../xds/client/XdsClientMetricReporter.java | 6 +- .../grpc/xds/GrpcXdsClientImplTestBase.java | 470 +++++++++--------- .../xds/XdsClientMetricReporterImplTest.java | 16 +- 9 files changed, 269 insertions(+), 318 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java b/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java index 2ab1d995aac..ac6e54bd180 100644 --- a/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java +++ b/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java @@ -52,6 +52,8 @@ final class SharedXdsClientPoolProvider implements XdsClientPoolFactory { private static final boolean LOG_XDS_NODE_ID = Boolean.parseBoolean( System.getenv("GRPC_LOG_XDS_NODE_ID")); private static final Logger log = Logger.getLogger(XdsClientImpl.class.getName()); + private static final ExponentialBackoffPolicy.Provider BACKOFF_POLICY_PROVIDER = + new ExponentialBackoffPolicy.Provider(); private final Bootstrapper bootstrapper; private final Object lock = new Object(); @@ -121,8 +123,6 @@ private static class SharedXdsClientPoolProviderHolder { @VisibleForTesting protected class RefCountedXdsClientObjectPool implements ObjectPool { - private final ExponentialBackoffPolicy.Provider backoffPolicyProvider = - new ExponentialBackoffPolicy.Provider(); private final BootstrapInfo bootstrapInfo; private final String target; // The target associated with the xDS client. private final XdsClientMetricReporterImpl xdsClientMetricReporter; @@ -139,7 +139,7 @@ protected class RefCountedXdsClientObjectPool implements ObjectPool { MetricRecorder metricRecorder) { this.bootstrapInfo = checkNotNull(bootstrapInfo); this.target = target; - this.xdsClientMetricReporter = new XdsClientMetricReporterImpl(metricRecorder); + this.xdsClientMetricReporter = new XdsClientMetricReporterImpl(metricRecorder, target); } @Override @@ -154,12 +154,11 @@ public XdsClient getObject() { DEFAULT_XDS_TRANSPORT_FACTORY, bootstrapInfo, scheduler, - backoffPolicyProvider, + BACKOFF_POLICY_PROVIDER, GrpcUtil.STOPWATCH_SUPPLIER, TimeProvider.SYSTEM_TIME_PROVIDER, MessagePrinter.INSTANCE, new TlsContextManagerImpl(bootstrapInfo), - getTarget(), xdsClientMetricReporter); xdsClientMetricReporter.setXdsClient(xdsClient); } diff --git a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java index 6361a42bc53..2f1eeb4439b 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java @@ -52,6 +52,7 @@ public class XdsClientMetricReporterImpl implements XdsClientMetricReporter { private static final LongGaugeMetricInstrument RESOURCES_GAUGE; private final MetricRecorder metricRecorder; + private final String target; @Nullable private Registration gaugeRegistration = null; @Nullable @@ -90,13 +91,14 @@ public class XdsClientMetricReporterImpl implements XdsClientMetricReporter { "grpc.xds.resource_type"), Collections.emptyList(), false); } - XdsClientMetricReporterImpl(MetricRecorder metricRecorder) { + XdsClientMetricReporterImpl(MetricRecorder metricRecorder, String target) { this.metricRecorder = metricRecorder; + this.target = target; } @Override public void reportResourceUpdates(long validResourceCount, long invalidResourceCount, - String target, String xdsServer, String resourceType) { + String xdsServer, String resourceType) { metricRecorder.addLongCounter(RESOURCE_UPDATES_VALID_COUNTER, validResourceCount, Arrays.asList(target, xdsServer, resourceType), Collections.emptyList()); metricRecorder.addLongCounter(RESOURCE_UPDATES_INVALID_COUNTER, invalidResourceCount, @@ -104,7 +106,7 @@ public void reportResourceUpdates(long validResourceCount, long invalidResourceC } @Override - public void reportServerFailure(long serverFailure, String target, String xdsServer) { + public void reportServerFailure(long serverFailure, String xdsServer) { metricRecorder.addLongCounter(SERVER_FAILURE_COUNTER, serverFailure, Arrays.asList(target, xdsServer), Collections.emptyList()); } @@ -127,7 +129,7 @@ void close() { } void reportCallbackMetrics(BatchRecorder recorder) { - MetricReporterCallback callback = new MetricReporterCallback(recorder); + MetricReporterCallback callback = new MetricReporterCallback(recorder, target); try { SettableFuture reportResourceCountsCompleted = this.xdsClient.reportResourceCounts( callback); @@ -148,21 +150,23 @@ void reportCallbackMetrics(BatchRecorder recorder) { static final class MetricReporterCallback implements ResourceCallback, ServerConnectionCallback { private final BatchRecorder recorder; + private final String target; - MetricReporterCallback(BatchRecorder recorder) { + MetricReporterCallback(BatchRecorder recorder, String target) { this.recorder = recorder; + this.target = target; } - // TODO(@dnvindhya): include the "authority" label once xds.authority is available. + // TODO(dnvindhya): include the "authority" label once xds.authority is available. @Override - public void reportResourceCountGauge(long resourceCount, String cacheState, String resourceType, - String target) { + public void reportResourceCountGauge(long resourceCount, String cacheState, + String resourceType) { recorder.recordLongGauge(RESOURCES_GAUGE, resourceCount, Arrays.asList(target, cacheState, resourceType), Collections.emptyList()); } @Override - public void reportServerConnectionGauge(boolean isConnected, String target, String xdsServer) { + public void reportServerConnectionGauge(boolean isConnected, String xdsServer) { recorder.recordLongGauge(CONNECTED_GAUGE, isConnected ? 1 : 0, Arrays.asList(target, xdsServer), Collections.emptyList()); } diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java index 40487dbfc89..c51709c174c 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java @@ -126,9 +126,9 @@ final class XdsNameResolver extends NameResolver { private final ConcurrentMap clusterRefs = new ConcurrentHashMap<>(); private final ConfigSelector configSelector = new ConfigSelector(); private final long randomChannelId; + private final MetricRecorder metricRecorder; private volatile RoutingConfig routingConfig = RoutingConfig.empty; - private volatile MetricRecorder metricRecorder; private Listener2 listener; private ObjectPool xdsClientPool; private XdsClient xdsClient; diff --git a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java index e1387935314..8c03e64f185 100644 --- a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java +++ b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java @@ -172,6 +172,8 @@ public void run() { private void internalStart() { try { + // TODO(dnvindhya): Add "#server" as "grpc.target" attribute value for + // xDS enabled servers. xdsClientPool = xdsClientPoolFactory.getOrCreate("", new MetricRecorder() {}); } catch (Exception e) { StatusException statusException = Status.UNAVAILABLE.withDescription( diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClient.java b/xds/src/main/java/io/grpc/xds/client/XdsClient.java index 424d3b99e3d..85aafa417c4 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClient.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClient.java @@ -381,14 +381,13 @@ public Map getServerLrsClientMap() { /** Callback used to report gauge metric value for resources. */ public interface ResourceCallback { - // TODO(@dnvindhya): include the "authority" label once xds.authority is available. - void reportResourceCountGauge(long resourceCount, String cacheState, String resourceType, - String target); + // TODO(dnvindhya): include the "authority" label once xds.authority is available. + void reportResourceCountGauge(long resourceCount, String cacheState, String resourceType); } /** Callback used to report a gauge metric value for server connections. */ public interface ServerConnectionCallback { - void reportServerConnectionGauge(boolean isConnected, String target, String xdsServer); + void reportServerConnectionGauge(boolean isConnected, String xdsServer); } /** diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java index 7bdf960b9b9..cf47c046165 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java @@ -101,7 +101,6 @@ public void uncaughtException(Thread t, Throwable e) { private final XdsLogger logger; private volatile boolean isShutdown; private final MessagePrettyPrinter messagePrinter; - private final String target; private final XdsClientMetricReporter metricReporter; public XdsClientImpl( @@ -113,7 +112,6 @@ public XdsClientImpl( TimeProvider timeProvider, MessagePrettyPrinter messagePrinter, Object securityConfig, - String target, XdsClientMetricReporter metricReporter) { this.xdsTransportFactory = xdsTransportFactory; this.bootstrapInfo = bootstrapInfo; @@ -123,7 +121,6 @@ public XdsClientImpl( this.timeProvider = timeProvider; this.messagePrinter = messagePrinter; this.securityConfig = securityConfig; - this.target = target; this.metricReporter = metricReporter; logId = InternalLogId.allocate("xds-client", null); logger = XdsLogger.withLogId(logId); @@ -148,7 +145,7 @@ private void handleStreamClosed(Status error, ServerInfo serverInfo) { syncContext.throwIfNotInThisSynchronizationContext(); cleanUpResourceTimers(); if (!error.isOk()) { - metricReporter.reportServerFailure(1L, target, serverInfo.target()); + metricReporter.reportServerFailure(1L, serverInfo.target()); for (Map> subscriberMap : resourceSubscribers.values()) { for (ResourceSubscriber subscriber : subscriberMap.values()) { @@ -474,7 +471,7 @@ private void handleResourceUpdate( Map> parsedResources = result.parsedResources; Set invalidResources = result.invalidResources; metricReporter.reportResourceUpdates(Long.valueOf(parsedResources.size()), - Long.valueOf(invalidResources.size()), target, + Long.valueOf(invalidResources.size()), args.getServerInfo().target(), xdsResourceType.typeUrl()); List errors = result.errors; @@ -539,7 +536,7 @@ public SettableFuture reportServerConnections(ServerConnectionCallback cal syncContext.execute(() -> { serverCpClientMap.forEach((serverInfo, controlPlaneClient) -> callback.reportServerConnectionGauge( - controlPlaneClient.hasWorkingAdsStream(), target, serverInfo.target())); + controlPlaneClient.hasWorkingAdsStream(), serverInfo.target())); future.set(null); }); return future; @@ -549,9 +546,17 @@ public SettableFuture reportServerConnections(ServerConnectionCallback cal public SettableFuture reportResourceCounts(ResourceCallback callback) { SettableFuture future = SettableFuture.create(); syncContext.execute(() -> { - Map, Map> resourceCountsByType = - getResourceCountsByType(); - reportResourceCountsToCallback(callback, resourceCountsByType); + for (XdsResourceType resourceType : resourceSubscribers.keySet()) { + Map resourceCountsByState = new HashMap<>(); + for (ResourceSubscriber subscriber : + resourceSubscribers.get(resourceType).values()) { + String cacheState = cacheStateFromResourceStatus(subscriber.metadata, + subscriber.data != null); + resourceCountsByState.compute(cacheState, (k, v) -> (v == null) ? 1 : v + 1); + } + resourceCountsByState.forEach((cacheState, count) -> + callback.reportResourceCountGauge(count, cacheState, resourceType.typeUrl())); + } future.set(null); }); return future; @@ -563,36 +568,6 @@ private String cacheStateFromResourceStatus(ResourceMetadata metadata, boolean i ? status + "_but_cached" : status; } - /** Calculates number of resources by ResourceType and ResourceSubscriber.metadata.status. */ - Map, Map> getResourceCountsByType() { - Map, Map> resourceCountsByType = new HashMap<>(); - for (XdsResourceType resourceType : resourceSubscribers.keySet()) { - Map resourceCountsByState = new HashMap<>(); - for (ResourceSubscriber subscriber : - resourceSubscribers.get(resourceType).values()) { - String cacheState = cacheStateFromResourceStatus(subscriber.metadata, - subscriber.data != null); - resourceCountsByState.compute(cacheState, (k, v) -> (v == null) ? 1 : v + 1); - } - resourceCountsByType.put(resourceType, resourceCountsByState); - } - return resourceCountsByType; - } - - /** Reports resource counts using the provided ResourceCallback. */ - void reportResourceCountsToCallback(ResourceCallback callback, - Map, Map> resourceCountsByType) { - for (Map.Entry, Map> entry : - resourceCountsByType.entrySet()) { - XdsResourceType resourceType = entry.getKey(); - Map resourceCountsByState = entry.getValue(); - // TODO(@dnvindhya): include the "authority" label once authority is available here. - resourceCountsByState.forEach((cacheState, count) -> - callback.reportResourceCountGauge(count, cacheState, resourceType.typeUrl(), target)); - } - } - - /** Tracks a single subscribed resource. */ private final class ResourceSubscriber { @Nullable private final ServerInfo serverInfo; diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClientMetricReporter.java b/xds/src/main/java/io/grpc/xds/client/XdsClientMetricReporter.java index 1657c1aae20..a044d501759 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClientMetricReporter.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClientMetricReporter.java @@ -29,22 +29,20 @@ public interface XdsClientMetricReporter { * * @param validResourceCount Number of resources that were valid. * @param invalidResourceCount Number of resources that were invalid. - * @param target Target of the gRPC channel. * @param xdsServer Target URI of the xDS server with which the XdsClient is communicating. * @param resourceType Type of XDS resource (e.g., "envoy.config.listener.v3.Listener"). */ default void reportResourceUpdates(long validResourceCount, long invalidResourceCount, - String target, String xdsServer, String resourceType) { + String xdsServer, String resourceType) { } /** * Reports number of xDS servers going from healthy to unhealthy. * * @param serverFailure Number of xDS server failures. - * @param target Target of the gRPC channel. * @param xdsServer Target URI of the xDS server with which the XdsClient is communicating. */ - default void reportServerFailure(long serverFailure, String target, String xdsServer) { + default void reportServerFailure(long serverFailure, String xdsServer) { } } diff --git a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java index d08fb5b5558..61dd113e25c 100644 --- a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java +++ b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java @@ -160,7 +160,6 @@ public abstract class GrpcXdsClientImplTestBase { private static final String VERSION_2 = "43"; private static final String VERSION_3 = "44"; private static final String NODE_ID = "cool-node-id"; - private static final String CHANNEL_TARGET = "target"; private static final Node NODE = Node.newBuilder().setId(NODE_ID).build(); private static final Any FAILING_ANY = MessageFactory.FAILING_ANY; private static final ChannelCredentials CHANNEL_CREDENTIALS = InsecureChannelCredentials.create(); @@ -382,7 +381,6 @@ public XdsTransport create(ServerInfo serverInfo) { timeProvider, MessagePrinter.INSTANCE, new TlsContextManagerImpl(bootstrapInfo), - CHANNEL_TARGET, xdsClientMetricReporter); assertThat(resourceDiscoveryCalls).isEmpty(); @@ -622,21 +620,18 @@ private void validateGoldenClusterLoadAssignment(EdsUpdate edsUpdate) { * resource count, and corresponding metric labels. */ private void verifyResourceValidInvalidCount(int times, long validResourceCount, - long invalidResourceCount, String dataPlaneTargetLabel, String xdsServerTargetLabel, + long invalidResourceCount, String xdsServerTargetLabel, String resourceType) { verify(xdsClientMetricReporter, times(times)).reportResourceUpdates( eq(validResourceCount), eq(invalidResourceCount), - eq(dataPlaneTargetLabel), eq(xdsServerTargetLabel), eq(resourceType)); } - private void verifyServerFailureCount(int times, long serverFailureCount, String dataPlaneTarget, - String xdsServer) { + private void verifyServerFailureCount(int times, long serverFailureCount, String xdsServer) { verify(xdsClientMetricReporter, times(times)).reportServerFailure( eq(serverFailureCount), - eq(dataPlaneTarget), eq(xdsServer)); } @@ -656,12 +651,11 @@ private void callback_ReportResourceCount() { } private void verifyResourceCountByCacheState(int times, long resourceCount, - String cacheState, String resourceTypeUrl, String dataPlaneTarget) { + String cacheState, String resourceTypeUrl) { verify(resourceCallback, times(times)).reportResourceCountGauge( eq(resourceCount), eq(cacheState), - eq(resourceTypeUrl), - eq(dataPlaneTarget)); + eq(resourceTypeUrl)); } /** @@ -679,11 +673,9 @@ private void callback_ReportServerConnection() { } } - private void verifyServerConnection(int times, boolean isConnected, String dataPlaneTarget, - String xdsServer) { + private void verifyServerConnection(int times, boolean isConnected, String xdsServer) { verify(serverConnectionCallback, times(times)).reportServerConnectionGauge( eq(isConnected), - eq(dataPlaneTarget), eq(xdsServer)); } @@ -703,7 +695,7 @@ public void ldsResourceNotFound() { verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); // Server failed to return subscribed resource within expected time window. fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); verify(ldsResourceWatcher).onResourceDoesNotExist(LDS_RESOURCE); @@ -712,7 +704,7 @@ public void ldsResourceNotFound() { // Check metric data. verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl()); } @Test @@ -791,7 +783,7 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid() { verifySubscribedResourcesMetadataSizes(3, 0, 0, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "requested", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 3, "requested", LDS.typeUrl()); // LDS -> {A, B, C}, version 1 ImmutableMap resourcesV1 = ImmutableMap.of( @@ -804,10 +796,9 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataAcked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); // Check metric data. - verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), - LDS.typeUrl()); + verifyResourceValidInvalidCount(1, 3, 0, xdsServerInfo.target(), LDS.typeUrl()); callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 3, "acked", LDS.typeUrl()); call.verifyRequest(LDS, subscribedResourceNames, VERSION_1, "0000", NODE); // LDS -> {A, B}, version 2 @@ -824,22 +815,21 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataNacked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, VERSION_2, TIME_INCREMENT * 2, errorsV2); // Check metric data. - verifyResourceValidInvalidCount(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), - LDS.typeUrl()); + verifyResourceValidInvalidCount(1, 1, 1, xdsServerInfo.target(), LDS.typeUrl()); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(LDS, "C"); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl()); } else { // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl()); } call.verifyRequestNack(LDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); @@ -851,21 +841,19 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid() { // {A} -> does not exist // {B, C} -> ACK, version 3 // Check metric data. - verifyResourceValidInvalidCount(1, 2, 0, CHANNEL_TARGET, xdsServerInfo.target(), - LDS.typeUrl()); + verifyResourceValidInvalidCount(1, 2, 0, xdsServerInfo.target(), LDS.typeUrl()); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(LDS, "A"); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "does_not_exist", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "does_not_exist", LDS.typeUrl()); + verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl()); } else { // When resource deletion is disabled, {A} stays ACKed in the previous version VERSION_2. verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 3, "acked", LDS.typeUrl(), - CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 3, "acked", LDS.typeUrl()); } verifyResourceMetadataAcked(LDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); verifyResourceMetadataAcked(LDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); @@ -893,8 +881,8 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti verifySubscribedResourcesMetadataSizes(3, 0, 3, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "requested", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 3, "requested", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 3, "requested", LDS.typeUrl()); + verifyResourceCountByCacheState(1, 3, "requested", RDS.typeUrl()); // LDS -> {A, B, C}, version 1 ImmutableMap resourcesV1 = ImmutableMap.of( @@ -903,15 +891,14 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti "C", Any.pack(mf.buildListenerWithApiListenerForRds("C", "C.1"))); call.sendResponse(LDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 - verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), - LDS.typeUrl()); + verifyResourceValidInvalidCount(1, 3, 0, xdsServerInfo.target(), LDS.typeUrl()); verifyResourceMetadataAcked(LDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(2, 3, "requested", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 3, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(2, 3, "requested", RDS.typeUrl()); call.verifyRequest(LDS, subscribedResourceNames, VERSION_1, "0000", NODE); // RDS -> {A.1, B.1, C.1}, version 1 @@ -926,11 +913,10 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti verifyResourceMetadataAcked(RDS, "B.1", resourcesV11.get("B.1"), VERSION_1, TIME_INCREMENT * 2); verifyResourceMetadataAcked(RDS, "C.1", resourcesV11.get("C.1"), VERSION_1, TIME_INCREMENT * 2); // Check metric data. - verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), - RDS.typeUrl()); + verifyResourceValidInvalidCount(1, 3, 0, xdsServerInfo.target(), RDS.typeUrl()); callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 3, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 3, "acked", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 3, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(1, 3, "acked", RDS.typeUrl()); // LDS -> {A, B}, version 2 // Failed to parse endpoint B @@ -942,8 +928,7 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> does not exist // Check metric data. - verifyResourceValidInvalidCount(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), - LDS.typeUrl()); + verifyResourceValidInvalidCount(1, 1, 1, xdsServerInfo.target(), LDS.typeUrl()); List errorsV2 = ImmutableList.of("LDS response Listener 'B' validation error: "); verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 3); verifyResourceMetadataNacked( @@ -953,18 +938,18 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti verifyResourceMetadataDoesNotExist(LDS, "C"); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(2, 3, "acked", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl()); + verifyResourceCountByCacheState(2, 3, "acked", RDS.typeUrl()); } else { // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(2, 3, "acked", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl()); + verifyResourceCountByCacheState(2, 3, "acked", RDS.typeUrl()); } call.verifyRequestNack(LDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); // {A.1} -> version 1 @@ -1032,7 +1017,7 @@ public void ldsResourceFound_containsRdsName() { ldsResourceWatcher); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); call.sendResponse(LDS, testListenerRds, VERSION_1, "0000"); // Client sends an ACK LDS request. @@ -1044,7 +1029,7 @@ public void ldsResourceFound_containsRdsName() { verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); } @Test @@ -1089,7 +1074,7 @@ public void ldsResourceUpdated() { verifyResourceMetadataRequested(LDS, LDS_RESOURCE); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); // Initial LDS response. call.sendResponse(LDS, testListenerVhosts, VERSION_1, "0000"); @@ -1099,7 +1084,7 @@ public void ldsResourceUpdated() { verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); // Updated LDS response. call.sendResponse(LDS, testListenerRds, VERSION_2, "0001"); @@ -1110,7 +1095,7 @@ public void ldsResourceUpdated() { verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl()); assertThat(channelForCustomAuthority).isNull(); assertThat(channelForEmptyAuthority).isNull(); } @@ -1122,7 +1107,7 @@ public void cancelResourceWatcherNotRemoveUrlSubscribers() { verifyResourceMetadataRequested(LDS, LDS_RESOURCE); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); // Initial LDS response. call.sendResponse(LDS, testListenerVhosts, VERSION_1, "0000"); @@ -1132,7 +1117,7 @@ public void cancelResourceWatcherNotRemoveUrlSubscribers() { verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); xdsClient.watchXdsResource(XdsListenerResource.getInstance(), LDS_RESOURCE + "1", ldsResourceWatcher); @@ -1150,7 +1135,7 @@ public void cancelResourceWatcherNotRemoveUrlSubscribers() { TIME_INCREMENT * 2); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl()); } @Test @@ -1217,7 +1202,7 @@ public void ldsResourceUpdated_withXdstpResourceName_withWrongType() { assertThat(channelForCustomAuthority).isNotNull(); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); String ldsResourceNameWithWrongType = "xdstp://authority.xds.com/envoy.config.route.v3.RouteConfiguration/listener1"; @@ -1231,7 +1216,7 @@ public void ldsResourceUpdated_withXdstpResourceName_withWrongType() { "Unsupported resource name: " + ldsResourceNameWithWrongType + " for type: LDS")); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "requested", LDS.typeUrl()); } @Test @@ -1351,7 +1336,7 @@ public void ldsResourceUpdate_withFaultInjection() { ldsResourceWatcher); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); Any listener = Any.pack( mf.buildListenerWithApiListener( LDS_RESOURCE, @@ -1392,7 +1377,7 @@ public void ldsResourceUpdate_withFaultInjection() { verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); LdsUpdate ldsUpdate = ldsUpdateCaptor.getValue(); assertThat(ldsUpdate.httpConnectionManager().virtualHosts()).hasSize(2); @@ -1425,7 +1410,7 @@ public void ldsResourceDeleted() { verifyResourceMetadataRequested(LDS, LDS_RESOURCE); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); // Initial LDS response. call.sendResponse(LDS, testListenerVhosts, VERSION_1, "0000"); @@ -1436,7 +1421,7 @@ public void ldsResourceDeleted() { verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); // Empty LDS response deletes the listener. call.sendResponse(LDS, Collections.emptyList(), VERSION_2, "0001"); @@ -1446,7 +1431,7 @@ public void ldsResourceDeleted() { verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl()); } /** @@ -1462,7 +1447,7 @@ public void ldsResourceDeleted_ignoreResourceDeletion() { verifyResourceMetadataRequested(LDS, LDS_RESOURCE); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); // Initial LDS response. call.sendResponse(LDS, testListenerVhosts, VERSION_1, "0000"); @@ -1473,7 +1458,7 @@ public void ldsResourceDeleted_ignoreResourceDeletion() { verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); // Empty LDS response does not delete the listener. call.sendResponse(LDS, Collections.emptyList(), VERSION_2, "0001"); @@ -1483,7 +1468,7 @@ public void ldsResourceDeleted_ignoreResourceDeletion() { verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl()); // onResourceDoesNotExist not called verify(ldsResourceWatcher, never()).onResourceDoesNotExist(LDS_RESOURCE); @@ -1498,7 +1483,7 @@ public void ldsResourceDeleted_ignoreResourceDeletion() { verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(3, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(3, 1, "acked", LDS.typeUrl()); verifyNoMoreInteractions(ldsResourceWatcher); } @@ -1519,7 +1504,7 @@ public void multipleLdsWatchers() { verifySubscribedResourcesMetadataSizes(2, 0, 0, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "requested", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 2, "requested", LDS.typeUrl()); fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); verify(ldsResourceWatcher).onResourceDoesNotExist(LDS_RESOURCE); @@ -1530,7 +1515,7 @@ public void multipleLdsWatchers() { verifySubscribedResourcesMetadataSizes(2, 0, 0, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "does_not_exist", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 2, "does_not_exist", LDS.typeUrl()); Any listenerTwo = Any.pack(mf.buildListenerWithApiListenerForRds(ldsResourceTwo, RDS_RESOURCE)); call.sendResponse(LDS, ImmutableList.of(testListenerVhosts, listenerTwo), VERSION_1, "0000"); @@ -1551,7 +1536,7 @@ public void multipleLdsWatchers() { verifySubscribedResourcesMetadataSizes(2, 0, 0, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl()); } @Test @@ -1620,7 +1605,7 @@ public void rdsResponseErrorHandling_nackWeightedSumZero() { verifyResourceMetadataRequested(RDS, RDS_RESOURCE); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl()); io.envoyproxy.envoy.config.route.v3.RouteAction routeAction = io.envoyproxy.envoy.config.route.v3.RouteAction.newBuilder() @@ -1658,7 +1643,7 @@ public void rdsResponseErrorHandling_nackWeightedSumZero() { verifySubscribedResourcesMetadataSizes(0, 0, 1, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "nacked", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "nacked", RDS.typeUrl()); // The response is NACKed with the same error message. call.verifyRequestNack(RDS, RDS_RESOURCE, "", "0000", NODE, errors); verify(rdsResourceWatcher, never()).onChanged(any(RdsUpdate.class)); @@ -1685,7 +1670,7 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { verifySubscribedResourcesMetadataSizes(0, 0, 3, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "requested", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 3, "requested", RDS.typeUrl()); // RDS -> {A, B, C}, version 1 List vhostsV1 = mf.buildOpaqueVirtualHosts(1); @@ -1699,9 +1684,9 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataAcked(RDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(RDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); // Check metric data. - verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), RDS.typeUrl()); + verifyResourceValidInvalidCount(1, 3, 0, xdsServerInfo.target(), RDS.typeUrl()); callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "acked", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 3, "acked", RDS.typeUrl()); call.verifyRequest(RDS, subscribedResourceNames, VERSION_1, "0000", NODE); // RDS -> {A, B}, version 2 @@ -1713,7 +1698,7 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { // {A} -> ACK, version 2 // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> ACK, version 1 - verifyResourceValidInvalidCount(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 1, 1, xdsServerInfo.target(), RDS.typeUrl()); List errorsV2 = ImmutableList.of("RDS response RouteConfiguration 'B' validation error: "); @@ -1723,8 +1708,8 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataAcked(RDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", RDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 2, "acked", RDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", RDS.typeUrl()); call.verifyRequestNack(RDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); // RDS -> {B, C} version 3 @@ -1735,7 +1720,7 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { call.sendResponse(RDS, resourcesV3.values().asList(), VERSION_3, "0002"); // {A} -> ACK, version 2 // {B, C} -> ACK, version 3 - verifyResourceValidInvalidCount(1, 2, 0, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 2, 0, xdsServerInfo.target(), RDS.typeUrl()); verifyResourceMetadataAcked(RDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataAcked(RDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); @@ -1743,7 +1728,7 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { call.verifyRequest(RDS, subscribedResourceNames, VERSION_3, "0002", NODE); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 3, "acked", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 3, "acked", RDS.typeUrl()); verifySubscribedResourcesMetadataSizes(0, 0, 3, 0); } @@ -1819,7 +1804,7 @@ public void rdsResourceUpdated() { verifyResourceMetadataRequested(RDS, RDS_RESOURCE); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl()); // Initial RDS response. call.sendResponse(RDS, testRouteConfig, VERSION_1, "0000"); @@ -1829,7 +1814,7 @@ public void rdsResourceUpdated() { verifyResourceMetadataAcked(RDS, RDS_RESOURCE, testRouteConfig, VERSION_1, TIME_INCREMENT); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", RDS.typeUrl()); // Updated RDS response. Any routeConfigUpdated = @@ -1844,7 +1829,7 @@ public void rdsResourceUpdated() { TIME_INCREMENT * 2); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "acked", RDS.typeUrl()); } @Test @@ -1858,8 +1843,8 @@ public void rdsResourceDeletedByLdsApiListener() { verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl()); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); call.sendResponse(LDS, testListenerRds, VERSION_1, "0000"); @@ -1870,8 +1855,8 @@ public void rdsResourceDeletedByLdsApiListener() { verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(2, 1, "requested", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(2, 1, "requested", RDS.typeUrl()); call.sendResponse(RDS, testRouteConfig, VERSION_1, "0000"); verify(rdsResourceWatcher).onChanged(rdsUpdateCaptor.capture()); @@ -1881,8 +1866,8 @@ public void rdsResourceDeletedByLdsApiListener() { verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "acked", RDS.typeUrl()); // The Listener is getting replaced configured with an RDS name, to the one configured with // vhosts. Expect the RDS resources to be discarded. @@ -1898,8 +1883,8 @@ public void rdsResourceDeletedByLdsApiListener() { verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(3, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(2, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(3, 1, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(2, 1, "acked", RDS.typeUrl()); } @Test @@ -1913,8 +1898,8 @@ public void rdsResourcesDeletedByLdsTcpListener() { verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl()); Message hcmFilter = mf.buildHttpConnectionManagerFilter( RDS_RESOURCE, null, Collections.singletonList(mf.buildTerminalFilter())); @@ -1941,8 +1926,8 @@ public void rdsResourcesDeletedByLdsTcpListener() { verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(2, 1, "requested", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(2, 1, "requested", RDS.typeUrl()); // Simulates receiving the requested RDS resource. call.sendResponse(RDS, testRouteConfig, VERSION_1, "0000"); @@ -1951,8 +1936,8 @@ public void rdsResourcesDeletedByLdsTcpListener() { verifyResourceMetadataAcked(RDS, RDS_RESOURCE, testRouteConfig, VERSION_1, TIME_INCREMENT * 2); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "acked", RDS.typeUrl()); // Simulates receiving an updated version of the requested LDS resource as a TCP listener // with a filter chain containing inlined RouteConfiguration. @@ -1981,8 +1966,8 @@ public void rdsResourcesDeletedByLdsTcpListener() { verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(3, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(2, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(3, 1, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(2, 1, "acked", RDS.typeUrl()); } @Test @@ -2116,7 +2101,7 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { verifySubscribedResourcesMetadataSizes(0, 3, 0, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 3, "requested", CDS.typeUrl()); // CDS -> {A, B, C}, version 1 ImmutableMap resourcesV1 = ImmutableMap.of( @@ -2131,14 +2116,14 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { ))); call.sendResponse(CDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 - verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 3, 0, xdsServerInfo.target(), CDS.typeUrl()); verifyResourceMetadataAcked(CDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(CDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(CDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "acked", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 3, "acked", CDS.typeUrl()); call.verifyRequest(CDS, subscribedResourceNames, VERSION_1, "0000", NODE); // CDS -> {A, B}, version 2 @@ -2152,7 +2137,7 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { // {A} -> ACK, version 2 // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> does not exist - verifyResourceValidInvalidCount(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 1, 1, xdsServerInfo.target(), CDS.typeUrl()); List errorsV2 = ImmutableList.of("CDS response Cluster 'B' validation error: "); verifyResourceMetadataAcked(CDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); @@ -2162,16 +2147,16 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataDoesNotExist(CDS, "C"); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "does_not_exist", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", CDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", CDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "does_not_exist", CDS.typeUrl()); } else { // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. verifyResourceMetadataAcked(CDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 2, "acked", CDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", CDS.typeUrl()); } call.verifyRequestNack(CDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); @@ -2186,20 +2171,20 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { call.sendResponse(CDS, resourcesV3.values().asList(), VERSION_3, "0002"); // {A} -> does not exit // {B, C} -> ACK, version 3 - verifyResourceValidInvalidCount(1, 2, 0, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 2, 0, xdsServerInfo.target(), CDS.typeUrl()); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(CDS, "A"); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(2, 1, "does_not_exist", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 2, "acked", CDS.typeUrl()); + verifyResourceCountByCacheState(2, 1, "does_not_exist", CDS.typeUrl()); } else { // When resource deletion is disabled, {A} stays ACKed in the previous version VERSION_2. verifyResourceMetadataAcked(CDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 3, "acked", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 3, "acked", CDS.typeUrl()); } verifyResourceMetadataAcked(CDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); verifyResourceMetadataAcked(CDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); @@ -2227,8 +2212,8 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti verifySubscribedResourcesMetadataSizes(0, 3, 0, 3); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "requested", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 3, "requested", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 3, "requested", CDS.typeUrl()); + verifyResourceCountByCacheState(1, 3, "requested", EDS.typeUrl()); // CDS -> {A, B, C}, version 1 ImmutableMap resourcesV1 = ImmutableMap.of( @@ -2243,15 +2228,15 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti ))); call.sendResponse(CDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 - verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 3, 0, xdsServerInfo.target(), CDS.typeUrl()); verifyResourceMetadataAcked(CDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(CDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(CDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "acked", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(2, 3, "requested", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 3, "acked", CDS.typeUrl()); + verifyResourceCountByCacheState(2, 3, "requested", EDS.typeUrl()); call.verifyRequest(CDS, subscribedResourceNames, VERSION_1, "0000", NODE); // EDS -> {A.1, B.1, C.1}, version 1 @@ -2263,15 +2248,15 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti "C.1", Any.pack(mf.buildClusterLoadAssignment("C.1", endpointsV1, dropOverloads))); call.sendResponse(EDS, resourcesV11.values().asList(), VERSION_1, "0000"); // {A.1, B.1, C.1} -> ACK, version 1 - verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), + verifyResourceValidInvalidCount(1, 3, 0, xdsServerInfo.target(), EDS.typeUrl()); verifyResourceMetadataAcked(EDS, "A.1", resourcesV11.get("A.1"), VERSION_1, TIME_INCREMENT * 2); verifyResourceMetadataAcked(EDS, "B.1", resourcesV11.get("B.1"), VERSION_1, TIME_INCREMENT * 2); verifyResourceMetadataAcked(EDS, "C.1", resourcesV11.get("C.1"), VERSION_1, TIME_INCREMENT * 2); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 3, "acked", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 3, "acked", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 3, "acked", CDS.typeUrl()); + verifyResourceCountByCacheState(1, 3, "acked", EDS.typeUrl()); // CDS -> {A, B}, version 2 // Failed to parse endpoint B @@ -2285,8 +2270,7 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> does not exist // Check metric data. - verifyResourceValidInvalidCount(1, 1, 1, - CHANNEL_TARGET, xdsServerInfo.target(), CDS.typeUrl()); + verifyResourceValidInvalidCount(1, 1, 1, xdsServerInfo.target(), CDS.typeUrl()); List errorsV2 = ImmutableList.of("CDS response Cluster 'B' validation error: "); verifyResourceMetadataAcked(CDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 3); verifyResourceMetadataNacked( @@ -2296,18 +2280,18 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti verifyResourceMetadataDoesNotExist(CDS, "C"); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "does_not_exist", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(2, 3, "acked", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", CDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", CDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "does_not_exist", CDS.typeUrl()); + verifyResourceCountByCacheState(2, 3, "acked", EDS.typeUrl()); } else { // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. verifyResourceMetadataAcked(CDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(2, 3, "acked", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 2, "acked", CDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", CDS.typeUrl()); + verifyResourceCountByCacheState(2, 3, "acked", EDS.typeUrl()); } call.verifyRequestNack(CDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); // {A.1} -> version 1 @@ -2553,7 +2537,7 @@ public void cdsResponseErrorHandling_badUpstreamTlsContext() { cdsResourceWatcher); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl()); // Management server sends back CDS response with UpstreamTlsContext. List clusters = ImmutableList.of(Any @@ -2573,7 +2557,7 @@ public void cdsResponseErrorHandling_badUpstreamTlsContext() { verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, errorMsg); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "requested", CDS.typeUrl()); } /** @@ -2586,7 +2570,7 @@ public void cdsResponseWithOutlierDetection() { cdsResourceWatcher); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl()); OutlierDetection outlierDetectionXds = OutlierDetection.newBuilder() .setInterval(Durations.fromNanos(100)) @@ -2648,7 +2632,7 @@ public void cdsResponseWithOutlierDetection() { verifySubscribedResourcesMetadataSizes(0, 1, 0, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", CDS.typeUrl()); } /** @@ -2662,7 +2646,7 @@ public void cdsResponseWithInvalidOutlierDetectionNacks() { cdsResourceWatcher); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl()); OutlierDetection outlierDetectionXds = OutlierDetection.newBuilder() .setMaxEjectionPercent(UInt32Value.of(101)).build(); @@ -2770,7 +2754,7 @@ public void cdsResponseErrorHandling_badTransportSocketName() { cdsResourceWatcher); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl()); // Management server sends back CDS response with UpstreamTlsContext. List clusters = ImmutableList.of(Any @@ -2786,7 +2770,7 @@ public void cdsResponseErrorHandling_badTransportSocketName() { call.verifyRequestNack(CDS, CDS_RESOURCE, "", "0000", NODE, ImmutableList.of(errorMsg)); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "requested", CDS.typeUrl()); verify(cdsResourceWatcher).onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, errorMsg); } @@ -2846,7 +2830,7 @@ public void cachedCdsResource_absent() { cdsResourceWatcher); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl()); fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); verify(cdsResourceWatcher).onResourceDoesNotExist(CDS_RESOURCE); ResourceWatcher watcher = mock(ResourceWatcher.class); @@ -2857,7 +2841,7 @@ public void cachedCdsResource_absent() { verifySubscribedResourcesMetadataSizes(0, 1, 0, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "does_not_exist", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "does_not_exist", CDS.typeUrl()); } @Test @@ -3141,7 +3125,7 @@ public void edsCleanupNonceAfterUnsubscription() { call.verifyRequest(EDS, "A.1", "", "", NODE); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", EDS.typeUrl()); // EDS -> {A.1}, version 1 List dropOverloads = ImmutableList.of(); @@ -3154,7 +3138,7 @@ public void edsCleanupNonceAfterUnsubscription() { verify(edsResourceWatcher, times(1)).onChanged(any()); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", EDS.typeUrl()); // trigger an EDS resource unsubscription. xdsClient.cancelXdsResourceWatch(XdsEndpointResource.getInstance(), "A.1", edsResourceWatcher); @@ -3167,7 +3151,7 @@ public void edsCleanupNonceAfterUnsubscription() { call.verifyRequest(EDS, "A.1", "", "", NODE, Mockito.timeout(2000).times(2)); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "requested", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "requested", EDS.typeUrl()); } @Test @@ -3231,7 +3215,7 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataRequested(EDS, "C"); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "requested", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 3, "requested", EDS.typeUrl()); // EDS -> {A, B, C}, version 1 List dropOverloads = ImmutableList.of(mf.buildDropOverload("lb", 200)); @@ -3242,14 +3226,13 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { "C", Any.pack(mf.buildClusterLoadAssignment("C", endpointsV1, dropOverloads))); call.sendResponse(EDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 - verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), - EDS.typeUrl()); + verifyResourceValidInvalidCount(1, 3, 0, xdsServerInfo.target(), EDS.typeUrl()); verifyResourceMetadataAcked(EDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(EDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(EDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "acked", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 3, "acked", EDS.typeUrl()); call.verifyRequest(EDS, subscribedResourceNames, VERSION_1, "0000", NODE); // EDS -> {A, B}, version 2 @@ -3263,8 +3246,7 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> ACK, version 1 // Check metric data. - verifyResourceValidInvalidCount(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), - EDS.typeUrl()); + verifyResourceValidInvalidCount(1, 1, 1, xdsServerInfo.target(), EDS.typeUrl()); List errorsV2 = ImmutableList.of("EDS response ClusterLoadAssignment 'B' validation error: "); verifyResourceMetadataAcked(EDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); @@ -3273,8 +3255,8 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataAcked(EDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", EDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 2, "acked", EDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", EDS.typeUrl()); call.verifyRequestNack(EDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); // EDS -> {B, C} version 3 @@ -3287,8 +3269,7 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { // {A} -> ACK, version 2 // {B, C} -> ACK, version 3 // Check metric data. - verifyResourceValidInvalidCount(1, 2, 0, CHANNEL_TARGET, xdsServerInfo.target(), - EDS.typeUrl()); + verifyResourceValidInvalidCount(1, 2, 0, xdsServerInfo.target(), EDS.typeUrl()); verifyResourceMetadataAcked(EDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataAcked(EDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); verifyResourceMetadataAcked(EDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); @@ -3296,7 +3277,7 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { verifySubscribedResourcesMetadataSizes(0, 0, 0, 3); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 3, "acked", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 3, "acked", EDS.typeUrl()); } @Test @@ -3378,7 +3359,7 @@ public void flowControlAbsent() throws Exception { verifyResourceMetadataRequested(CDS, anotherCdsResource); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 2, "requested", CDS.typeUrl()); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); call.verifyRequest(CDS, Arrays.asList(CDS_RESOURCE, anotherCdsResource), "", "", NODE); @@ -3388,8 +3369,8 @@ public void flowControlAbsent() throws Exception { CDS, CDS_RESOURCE, testClusterRoundRobin, VERSION_1, TIME_INCREMENT); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", CDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl()); call.verifyRequest(CDS, Arrays.asList(CDS_RESOURCE, anotherCdsResource), VERSION_1, "0000", NODE); verifyNoInteractions(cdsResourceWatcher, anotherWatcher); @@ -3419,8 +3400,8 @@ public void flowControlAbsent() throws Exception { CDS, CDS_RESOURCE, testClusterRoundRobin, VERSION_1, TIME_INCREMENT); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "acked", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "does_not_exist", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "acked", CDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "does_not_exist", CDS.typeUrl()); barrier.await(); verify(cdsResourceWatcher, atLeastOnce()).onChanged(any()); String errorMsg = "CDS response Cluster 'cluster.googleapis.com2' validation error: " @@ -3434,8 +3415,8 @@ public void flowControlAbsent() throws Exception { verify(anotherWatcher).onError(any()); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(3, 1, "acked", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "nacked", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(3, 1, "acked", CDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "nacked", CDS.typeUrl()); } private Answer blockUpdate(CyclicBarrier barrier) { @@ -3587,8 +3568,8 @@ public void edsResourceDeletedByCds() { verifySubscribedResourcesMetadataSizes(0, 2, 0, 2); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "requested", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 2, "requested", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 2, "requested", CDS.typeUrl()); + verifyResourceCountByCacheState(1, 2, "requested", EDS.typeUrl()); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); List clusters = ImmutableList.of( @@ -3612,8 +3593,8 @@ public void edsResourceDeletedByCds() { verifyResourceMetadataRequested(EDS, resource); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(2, 2, "requested", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 2, "acked", CDS.typeUrl()); + verifyResourceCountByCacheState(2, 2, "requested", EDS.typeUrl()); List clusterLoadAssignments = ImmutableList.of( @@ -3646,8 +3627,8 @@ public void edsResourceDeletedByCds() { verifySubscribedResourcesMetadataSizes(0, 2, 0, 2); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 2, "acked", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 2, "acked", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 2, "acked", CDS.typeUrl()); + verifyResourceCountByCacheState(1, 2, "acked", EDS.typeUrl()); clusters = ImmutableList.of( Any.pack(mf.buildEdsCluster(resource, null, "round_robin", null, null, true, null, @@ -3671,8 +3652,8 @@ public void edsResourceDeletedByCds() { verifyResourceMetadataAcked(CDS, CDS_RESOURCE, clusters.get(1), VERSION_2, TIME_INCREMENT * 3); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(3, 2, "acked", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(2, 2, "acked", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(3, 2, "acked", CDS.typeUrl()); + verifyResourceCountByCacheState(2, 2, "acked", EDS.typeUrl()); } @Test @@ -3774,12 +3755,12 @@ public void streamClosedWithNoResponse() { DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(1, true, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(1, true, xdsServerInfo.target()); // Management server closes the RPC stream before sending any response. call.sendCompleted(); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(1, false, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(1, false, xdsServerInfo.target()); verify(ldsResourceWatcher, Mockito.timeout(1000).times(1)) .onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, @@ -3797,7 +3778,7 @@ public void streamClosedAfterSendingResponses() { DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(1, true, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(1, true, xdsServerInfo.target()); ScheduledTask ldsResourceTimeout = Iterables.getOnlyElement(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); ScheduledTask rdsResourceTimeout = @@ -3805,7 +3786,7 @@ public void streamClosedAfterSendingResponses() { call.sendResponse(LDS, testListenerRds, VERSION_1, "0000"); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(2, true, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(2, true, xdsServerInfo.target()); assertThat(ldsResourceTimeout.isCancelled()).isTrue(); call.sendResponse(RDS, testRouteConfig, VERSION_1, "0000"); assertThat(rdsResourceTimeout.isCancelled()).isTrue(); @@ -3813,7 +3794,7 @@ public void streamClosedAfterSendingResponses() { call.sendCompleted(); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(3, true, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(3, true, xdsServerInfo.target()); verify(ldsResourceWatcher, never()).onError(errorCaptor.capture()); verify(rdsResourceWatcher, never()).onError(errorCaptor.capture()); } @@ -3824,7 +3805,7 @@ public void streamClosedAndRetryWithBackoff() { xdsClient.watchXdsResource(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(1, true, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(1, true, xdsServerInfo.target()); xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, rdsResourceWatcher); xdsClient.watchXdsResource(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); @@ -3849,7 +3830,7 @@ public void streamClosedAndRetryWithBackoff() { // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(1, false, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(1, false, xdsServerInfo.target()); // Retry after backoff. inOrder.verify(backoffPolicyProvider).get(); @@ -3866,7 +3847,7 @@ public void streamClosedAndRetryWithBackoff() { // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(2, false, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(2, false, xdsServerInfo.target()); // Management server becomes unreachable. String errorMsg = "my fault"; @@ -3882,7 +3863,7 @@ public void streamClosedAndRetryWithBackoff() { // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(3, false, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(3, false, xdsServerInfo.target()); // Retry after backoff. inOrder.verify(backoffPolicy1).nextBackoffNanos(); @@ -3903,7 +3884,7 @@ public void streamClosedAndRetryWithBackoff() { call.verifyRequest(LDS, LDS_RESOURCE, "63", "3242", NODE); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(2, true, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(2, true, xdsServerInfo.target()); List routeConfigs = ImmutableList.of( Any.pack(mf.buildRouteConfiguration(RDS_RESOURCE, mf.buildOpaqueVirtualHosts(2)))); @@ -3920,7 +3901,7 @@ public void streamClosedAndRetryWithBackoff() { // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(3, true, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(3, true, xdsServerInfo.target()); // Reset backoff sequence and retry after backoff. inOrder.verify(backoffPolicyProvider).get(); @@ -3946,7 +3927,7 @@ public void streamClosedAndRetryWithBackoff() { // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(4, false, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(4, false, xdsServerInfo.target()); // Retry after backoff. inOrder.verify(backoffPolicy2).nextBackoffNanos(); @@ -3962,7 +3943,7 @@ public void streamClosedAndRetryWithBackoff() { // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(5, false, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(5, false, xdsServerInfo.target()); inOrder.verifyNoMoreInteractions(); } @@ -3976,7 +3957,7 @@ public void streamClosedAndRetryRaceWithAddRemoveWatchers() { DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(1, true, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(1, true, xdsServerInfo.target()); call.sendError(Status.UNAVAILABLE.asException()); verify(ldsResourceWatcher, Mockito.timeout(1000).times(1)) .onError(errorCaptor.capture()); @@ -3989,7 +3970,7 @@ public void streamClosedAndRetryRaceWithAddRemoveWatchers() { // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(1, false, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(1, false, xdsServerInfo.target()); xdsClient.cancelXdsResourceWatch(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); @@ -4007,7 +3988,7 @@ public void streamClosedAndRetryRaceWithAddRemoveWatchers() { // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(2,false, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(2,false, xdsServerInfo.target()); call.sendResponse(LDS, testListenerRds, VERSION_1, "0000"); List routeConfigs = ImmutableList.of( @@ -4016,7 +3997,7 @@ public void streamClosedAndRetryRaceWithAddRemoveWatchers() { // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(2, true, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(2, true, xdsServerInfo.target()); verifyNoMoreInteractions(ldsResourceWatcher, rdsResourceWatcher); } @@ -4031,12 +4012,12 @@ public void streamClosedAndRetryRestartsResourceInitialFetchTimerForUnresolvedRe DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(1, true, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(1, true, xdsServerInfo.target()); callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "requested", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "requested", EDS.typeUrl()); ScheduledTask ldsResourceTimeout = Iterables.getOnlyElement(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); ScheduledTask rdsResourceTimeout = @@ -4049,23 +4030,23 @@ public void streamClosedAndRetryRestartsResourceInitialFetchTimerForUnresolvedRe assertThat(ldsResourceTimeout.isCancelled()).isTrue(); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(2, true, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(2, true, xdsServerInfo.target()); callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(2, 1, "requested", RDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(2, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(2, 1, "requested", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(2, 1, "requested", RDS.typeUrl()); + verifyResourceCountByCacheState(2, 1, "requested", CDS.typeUrl()); + verifyResourceCountByCacheState(2, 1, "requested", EDS.typeUrl()); call.sendResponse(RDS, testRouteConfig, VERSION_1, "0000"); assertThat(rdsResourceTimeout.isCancelled()).isTrue(); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(3, true, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(3, true, xdsServerInfo.target()); callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(3, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(3, 1, "requested", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "acked", RDS.typeUrl()); + verifyResourceCountByCacheState(3, 1, "requested", CDS.typeUrl()); + verifyResourceCountByCacheState(3, 1, "requested", EDS.typeUrl()); call.sendError(Status.UNAVAILABLE.asException()); assertThat(cdsResourceTimeout.isCancelled()).isTrue(); @@ -4076,12 +4057,12 @@ public void streamClosedAndRetryRestartsResourceInitialFetchTimerForUnresolvedRe verify(edsResourceWatcher, never()).onError(errorCaptor.capture()); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(4, true, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(4, true, xdsServerInfo.target()); callback_ReportResourceCount(); - verifyResourceCountByCacheState(3, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(2, 1, "acked", RDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(4, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(4, 1, "requested", EDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(3, 1, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(2, 1, "acked", RDS.typeUrl()); + verifyResourceCountByCacheState(4, 1, "requested", CDS.typeUrl()); + verifyResourceCountByCacheState(4, 1, "requested", EDS.typeUrl()); fakeClock.forwardNanos(10L); assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(0); @@ -4245,10 +4226,10 @@ public void sendingToStoppedServer() throws Exception { fakeClock.forwardTime(14, TimeUnit.SECONDS); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(1, false, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(1, false, xdsServerInfo.target()); callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "unknown", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "unknown", LDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl()); // Restart the server xdsServer = cleanupRule.register( @@ -4265,10 +4246,10 @@ public void sendingToStoppedServer() throws Exception { DiscoveryRpcCall call = resourceDiscoveryCalls.poll(3, TimeUnit.SECONDS); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(2, false, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(2, false, xdsServerInfo.target()); callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "unknown", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(2, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 1, "unknown", LDS.typeUrl()); + verifyResourceCountByCacheState(2, 1, "requested", CDS.typeUrl()); if (call == null) { // The first rpcRetry may have happened before the channel was ready fakeClock.forwardTime(50, TimeUnit.SECONDS); call = resourceDiscoveryCalls.poll(3, TimeUnit.SECONDS); @@ -4276,10 +4257,10 @@ public void sendingToStoppedServer() throws Exception { // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(3, false, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(3, false, xdsServerInfo.target()); callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(3, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); + verifyResourceCountByCacheState(3, 1, "requested", CDS.typeUrl()); // NOTE: There is a ScheduledExecutorService that may get involved due to the reconnect // so you cannot rely on the logic being single threaded. The timeout() in verifyRequest @@ -4294,10 +4275,10 @@ public void sendingToStoppedServer() throws Exception { verifySubscribedResourcesMetadataSizes(1, 1, 0, 0); // Check metric data. callback_ReportServerConnection(); - verifyServerConnection(1, true, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerConnection(1, true, xdsServerInfo.target()); callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(4, 1, "requested", CDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(4, 1, "requested", CDS.typeUrl()); } catch (Throwable t) { throw t; // This allows putting a breakpoint here for debugging } @@ -4353,8 +4334,7 @@ public void validAndInvalidResourceMetricReport() { ))); call.sendResponse(CDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 - verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), - CDS.typeUrl()); + verifyResourceValidInvalidCount(1, 3, 0, xdsServerInfo.target(), CDS.typeUrl()); // EDS -> {A.1, B.1, C.1}, version 1 List dropOverloads = ImmutableList.of(); @@ -4365,8 +4345,7 @@ public void validAndInvalidResourceMetricReport() { "C.1", Any.pack(mf.buildClusterLoadAssignment("C.1", endpointsV1, dropOverloads))); call.sendResponse(EDS, resourcesV11.values().asList(), VERSION_1, "0000"); // {A.1, B.1, C.1} -> ACK, version 1 - verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), - EDS.typeUrl()); + verifyResourceValidInvalidCount(1, 3, 0, xdsServerInfo.target(), EDS.typeUrl()); // CDS -> {A, B}, version 2 // Failed to parse endpoint B @@ -4379,8 +4358,7 @@ public void validAndInvalidResourceMetricReport() { // {A} -> ACK, version 2 // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> does not exist - verifyResourceValidInvalidCount(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), - CDS.typeUrl()); + verifyResourceValidInvalidCount(1, 1, 1, xdsServerInfo.target(), CDS.typeUrl()); } @Test @@ -4398,7 +4376,7 @@ public void serverFailureMetricReport() { verify(rdsResourceWatcher).onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, "ADS stream closed with OK before receiving a response"); - verifyServerFailureCount(1, 1, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerFailureCount(1, 1, xdsServerInfo.target()); } @Test @@ -4413,7 +4391,7 @@ public void serverFailureMetricReport_forRetryAndBackoff() { // Management server closes the RPC stream with an error. call.sendError(Status.UNKNOWN.asException()); - verifyServerFailureCount(1, 1, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerFailureCount(1, 1, xdsServerInfo.target()); // Retry after backoff. inOrder.verify(backoffPolicyProvider).get(); @@ -4427,7 +4405,7 @@ public void serverFailureMetricReport_forRetryAndBackoff() { // Management server becomes unreachable. String errorMsg = "my fault"; call.sendError(Status.UNAVAILABLE.withDescription(errorMsg).asException()); - verifyServerFailureCount(2, 1, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerFailureCount(2, 1, xdsServerInfo.target()); // Retry after backoff. inOrder.verify(backoffPolicy1).nextBackoffNanos(); @@ -4446,7 +4424,7 @@ public void serverFailureMetricReport_forRetryAndBackoff() { call.sendError(Status.DEADLINE_EXCEEDED.asException()); // Server Failure metric will not be reported, as stream is closed with an error after receiving // a response - verifyServerFailureCount(2, 1, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerFailureCount(2, 1, xdsServerInfo.target()); // Reset backoff sequence and retry after backoff. inOrder.verify(backoffPolicyProvider).get(); @@ -4459,7 +4437,7 @@ public void serverFailureMetricReport_forRetryAndBackoff() { // Management server becomes unreachable again. call.sendError(Status.UNAVAILABLE.asException()); - verifyServerFailureCount(3, 1, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerFailureCount(3, 1, xdsServerInfo.target()); // Retry after backoff. inOrder.verify(backoffPolicy2).nextBackoffNanos(); @@ -4474,7 +4452,7 @@ public void serverFailureMetricReport_forRetryAndBackoff() { call.sendCompleted(); // Server Failure metric will not be reported once again, as stream is closed after receiving a // response - verifyServerFailureCount(3, 1, CHANNEL_TARGET, xdsServerInfo.target()); + verifyServerFailureCount(3, 1, xdsServerInfo.target()); } @@ -4492,7 +4470,7 @@ public void testReportResourceCounts() { verifySubscribedResourcesMetadataSizes(3, 0, 0, 0); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "requested", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 3, "requested", LDS.typeUrl()); // LDS -> {A, B, C}, version 1 ImmutableMap resourcesV1 = ImmutableMap.of( @@ -4501,14 +4479,13 @@ public void testReportResourceCounts() { "C", Any.pack(mf.buildListenerWithApiListenerForRds("C", "C.1"))); call.sendResponse(LDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 - verifyResourceValidInvalidCount(1, 3, 0, CHANNEL_TARGET, xdsServerInfo.target(), - LDS.typeUrl()); + verifyResourceValidInvalidCount(1, 3, 0, xdsServerInfo.target(), LDS.typeUrl()); verifyResourceMetadataAcked(LDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 3, "acked", LDS.typeUrl()); call.verifyRequest(LDS, subscribedResourceNames, VERSION_1, "0000", NODE); // LDS -> {A, B}, version 2 @@ -4520,8 +4497,7 @@ public void testReportResourceCounts() { // {A} -> ACK, version 2 // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> does not exist - verifyResourceValidInvalidCount(1, 1, 1, CHANNEL_TARGET, xdsServerInfo.target(), - LDS.typeUrl()); + verifyResourceValidInvalidCount(1, 1, 1, xdsServerInfo.target(), LDS.typeUrl()); List errorsV2 = ImmutableList.of("LDS response Listener 'B' validation error: "); verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataNacked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, @@ -4530,16 +4506,16 @@ public void testReportResourceCounts() { verifyResourceMetadataDoesNotExist(LDS, "C"); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl()); } else { // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl()); } call.verifyRequestNack(LDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); @@ -4550,20 +4526,19 @@ public void testReportResourceCounts() { call.sendResponse(LDS, resourcesV3.values().asList(), VERSION_3, "0002"); // {A} -> does not exist // {B, C} -> ACK, version 3 - verifyResourceValidInvalidCount(1, 2, 0, CHANNEL_TARGET, xdsServerInfo.target(), - LDS.typeUrl()); + verifyResourceValidInvalidCount(1, 2, 0, xdsServerInfo.target(), LDS.typeUrl()); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(LDS, "A"); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl(), CHANNEL_TARGET); - verifyResourceCountByCacheState(2, 1, "does_not_exist", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(2, 1, "does_not_exist", LDS.typeUrl()); } else { // When resource deletion is disabled, {A} stays ACKed in the previous version VERSION_2. verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); // Check metric data. callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 3, "acked", LDS.typeUrl(), CHANNEL_TARGET); + verifyResourceCountByCacheState(2, 3, "acked", LDS.typeUrl()); } verifyResourceMetadataAcked(LDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); verifyResourceMetadataAcked(LDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); @@ -4582,7 +4557,6 @@ private XdsClientImpl createXdsClient(String serverUri) { timeProvider, MessagePrinter.INSTANCE, new TlsContextManagerImpl(bootstrapInfo), - CHANNEL_TARGET, xdsClientMetricReporter); } diff --git a/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java b/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java index b69e7c7c27a..ef4f2238ded 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java @@ -85,13 +85,13 @@ public class XdsClientMetricReporterImplTest { @Before public void setUp() { - reporter = new XdsClientMetricReporterImpl(mockMetricRecorder); + reporter = new XdsClientMetricReporterImpl(mockMetricRecorder, target); } @Test public void reportResourceUpdates() { // TODO(dnvindhya): add the "authority" label once available. - reporter.reportResourceUpdates(10, 5, target, server, resourceTypeUrl); + reporter.reportResourceUpdates(10, 5, server, resourceTypeUrl); verify(mockMetricRecorder).addLongCounter( eqMetricInstrumentName("grpc.xds_client.resource_updates_valid"), eq((long) 10), eq(Lists.newArrayList(target, server, resourceTypeUrl)), @@ -105,7 +105,7 @@ public void reportResourceUpdates() { @Test public void reportServerFailure() { - reporter.reportServerFailure(1, target, server); + reporter.reportServerFailure(1, server); verify(mockMetricRecorder).addLongCounter( eqMetricInstrumentName("grpc.xds_client.server_failure"), eq((long) 1), eq(Lists.newArrayList(target, server)), @@ -200,11 +200,11 @@ public void metricGauges() { // Verify that reportResourceCounts and reportServerConnections were called // with the captured callback - callback.reportResourceCountGauge(10, "acked", resourceTypeUrl, target); + callback.reportResourceCountGauge(10, "acked", resourceTypeUrl); inOrder.verify(mockBatchRecorder) .recordLongGauge(eqMetricInstrumentName("grpc.xds_client.resources"), eq(10L), any(), any()); - callback.reportServerConnectionGauge(true, target, "xdsServer"); + callback.reportServerConnectionGauge(true, "xdsServer"); inOrder.verify(mockBatchRecorder) .recordLongGauge(eqMetricInstrumentName("grpc.xds_client.connected"), eq(1L), any(), any()); @@ -214,16 +214,16 @@ public void metricGauges() { @Test public void metricReporterCallback() { MetricReporterCallback callback = - new MetricReporterCallback(mockBatchRecorder); + new MetricReporterCallback(mockBatchRecorder, target); - callback.reportServerConnectionGauge(true, target, server); + callback.reportServerConnectionGauge(true, server); verify(mockBatchRecorder, times(1)).recordLongGauge( eqMetricInstrumentName("grpc.xds_client.connected"), eq(1L), eq(Lists.newArrayList(target, server)), eq(Lists.newArrayList())); String cacheState = "requested"; - callback.reportResourceCountGauge(10, cacheState, resourceTypeUrl, target); + callback.reportResourceCountGauge(10, cacheState, resourceTypeUrl); verify(mockBatchRecorder, times(1)).recordLongGauge( eqMetricInstrumentName("grpc.xds_client.resources"), eq(10L), eq(Arrays.asList(target, cacheState, resourceTypeUrl)), From 3a283e4acdd7443759cb6a04864e674e13ab7a8a Mon Sep 17 00:00:00 2001 From: Vindhya Ningegowda Date: Tue, 19 Nov 2024 17:11:45 -0800 Subject: [PATCH 08/13] Moved resource count metric reporting from XdsClientImpl to XdsClientMetricReporterImpl and addressed review comments --- .../grpc/xds/SharedXdsClientPoolProvider.java | 16 +- .../grpc/xds/XdsClientMetricReporterImpl.java | 71 +++- .../grpc/xds/client/ControlPlaneClient.java | 8 +- .../java/io/grpc/xds/client/XdsClient.java | 43 +- .../io/grpc/xds/client/XdsClientImpl.java | 37 +- .../grpc/xds/GrpcXdsClientImplTestBase.java | 402 ++---------------- .../xds/SharedXdsClientPoolProviderTest.java | 2 +- .../xds/XdsClientMetricReporterImplTest.java | 209 +++++++-- 8 files changed, 300 insertions(+), 488 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java b/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java index ac6e54bd180..779349744ff 100644 --- a/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java +++ b/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java @@ -121,11 +121,11 @@ private static class SharedXdsClientPoolProviderHolder { @ThreadSafe @VisibleForTesting - protected class RefCountedXdsClientObjectPool implements ObjectPool { + class RefCountedXdsClientObjectPool implements ObjectPool { private final BootstrapInfo bootstrapInfo; private final String target; // The target associated with the xDS client. - private final XdsClientMetricReporterImpl xdsClientMetricReporter; + private final MetricRecorder metricRecorder; private final Object lock = new Object(); @GuardedBy("lock") private ScheduledExecutorService scheduler; @@ -133,13 +133,15 @@ protected class RefCountedXdsClientObjectPool implements ObjectPool { private XdsClient xdsClient; @GuardedBy("lock") private int refCount; + @GuardedBy("lock") + private XdsClientMetricReporterImpl metricReporter; @VisibleForTesting RefCountedXdsClientObjectPool(BootstrapInfo bootstrapInfo, String target, MetricRecorder metricRecorder) { this.bootstrapInfo = checkNotNull(bootstrapInfo); this.target = target; - this.xdsClientMetricReporter = new XdsClientMetricReporterImpl(metricRecorder, target); + this.metricRecorder = metricRecorder; } @Override @@ -150,6 +152,7 @@ public XdsClient getObject() { log.log(Level.INFO, "xDS node ID: {0}", bootstrapInfo.node().getId()); } scheduler = SharedResourceHolder.get(GrpcUtil.TIMER_SERVICE); + metricReporter = new XdsClientMetricReporterImpl(metricRecorder, target); xdsClient = new XdsClientImpl( DEFAULT_XDS_TRANSPORT_FACTORY, bootstrapInfo, @@ -159,8 +162,8 @@ public XdsClient getObject() { TimeProvider.SYSTEM_TIME_PROVIDER, MessagePrinter.INSTANCE, new TlsContextManagerImpl(bootstrapInfo), - xdsClientMetricReporter); - xdsClientMetricReporter.setXdsClient(xdsClient); + metricReporter); + metricReporter.setXdsClient(xdsClient); } refCount++; return xdsClient; @@ -174,7 +177,8 @@ public XdsClient returnObject(Object object) { if (refCount == 0) { xdsClient.shutdown(); xdsClient = null; - xdsClientMetricReporter.close(); + metricReporter.close(); + metricReporter = null; targetToXdsClientMap.remove(target); scheduler = SharedResourceHolder.release(GrpcUtil.TIMER_SERVICE, scheduler); } diff --git a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java index 2f1eeb4439b..e085bdba74c 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java @@ -17,6 +17,7 @@ package io.grpc.xds; import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import io.grpc.Internal; import io.grpc.LongCounterMetricInstrument; @@ -28,11 +29,19 @@ import io.grpc.MetricRecorder.Registration; import io.grpc.xds.client.XdsClient; import io.grpc.xds.client.XdsClient.ResourceCallback; +import io.grpc.xds.client.XdsClient.ResourceMetadata; +import io.grpc.xds.client.XdsClient.ResourceMetadata.ResourceMetadataStatus; import io.grpc.xds.client.XdsClient.ServerConnectionCallback; import io.grpc.xds.client.XdsClientMetricReporter; +import io.grpc.xds.client.XdsResourceType; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.logging.Level; import java.util.logging.Logger; import javax.annotation.Nullable; @@ -41,7 +50,7 @@ * XdsClientMetricReporter implementation. */ @Internal -public class XdsClientMetricReporterImpl implements XdsClientMetricReporter { +final class XdsClientMetricReporterImpl implements XdsClientMetricReporter { private static final Logger logger = Logger.getLogger( XdsClientMetricReporterImpl.class.getName()); @@ -55,8 +64,6 @@ public class XdsClientMetricReporterImpl implements XdsClientMetricReporter { private final String target; @Nullable private Registration gaugeRegistration = null; - @Nullable - private XdsClient xdsClient = null; static { MetricInstrumentRegistry metricInstrumentRegistry @@ -111,13 +118,13 @@ public void reportServerFailure(long serverFailure, String xdsServer) { Arrays.asList(target, xdsServer), Collections.emptyList()); } - void setXdsClient(XdsClient client) { - this.xdsClient = client; + void setXdsClient(XdsClient xdsClient) { + assert gaugeRegistration == null; // register gauge here this.gaugeRegistration = metricRecorder.registerBatchCallback(new BatchCallback() { @Override public void accept(BatchRecorder recorder) { - reportCallbackMetrics(recorder); + reportCallbackMetrics(recorder, xdsClient); } }, CONNECTED_GAUGE, RESOURCES_GAUGE); } @@ -125,20 +132,29 @@ public void accept(BatchRecorder recorder) { void close() { if (gaugeRegistration != null) { gaugeRegistration.close(); + gaugeRegistration = null; } } - void reportCallbackMetrics(BatchRecorder recorder) { + void reportCallbackMetrics(BatchRecorder recorder, XdsClient xdsClient) { MetricReporterCallback callback = new MetricReporterCallback(recorder, target); try { - SettableFuture reportResourceCountsCompleted = this.xdsClient.reportResourceCounts( - callback); SettableFuture reportServerConnectionsCompleted = - this.xdsClient.reportServerConnections(callback); + xdsClient.reportServerConnections(callback); + + ListenableFuture, Map>> + getResourceMetadataCompleted = xdsClient.getSubscribedResourcesMetadataSnapshot(); + + Map, Map> metadataByType = + getResourceMetadataCompleted.get(10, TimeUnit.SECONDS); + + SettableFuture reportResourceCountsCompleted = computeAndReportResourceCounts( + metadataByType, callback); + // Normally this shouldn't take long, but adding a timeout to avoid indefinite blocking - Void unused1 = reportResourceCountsCompleted.get(5, TimeUnit.SECONDS); - Void unused2 = reportServerConnectionsCompleted.get(5, TimeUnit.SECONDS); - } catch (Exception e) { + Void unused1 = reportServerConnectionsCompleted.get(5, TimeUnit.SECONDS); + Void unused2 = reportResourceCountsCompleted.get(5, TimeUnit.SECONDS); + } catch (ExecutionException | TimeoutException | InterruptedException e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); // re-set the current thread's interruption state } @@ -146,6 +162,35 @@ void reportCallbackMetrics(BatchRecorder recorder) { } } + private SettableFuture computeAndReportResourceCounts( + Map, Map> metadataByType, + MetricReporterCallback callback) { + SettableFuture future = SettableFuture.create(); + + for (Map.Entry, Map> metadataByTypeEntry : + metadataByType.entrySet()) { + XdsResourceType type = metadataByTypeEntry.getKey(); + + Map resourceCountsByState = new HashMap<>(); + for (ResourceMetadata metadata : metadataByTypeEntry.getValue().values()) { + String cacheState = cacheStateFromResourceStatus(metadata.getStatus(), metadata.isCached()); + resourceCountsByState.compute(cacheState, (k, v) -> (v == null) ? 1 : v + 1); + } + + resourceCountsByState.forEach((cacheState, count) -> + callback.reportResourceCountGauge(count, cacheState, type.typeUrl())); + } + future.set(null); + return future; + } + + private static String cacheStateFromResourceStatus(ResourceMetadataStatus metadataStatus, + boolean isResourceCached) { + String status = metadataStatus.toString().toLowerCase(Locale.ROOT); + return metadataStatus == ResourceMetadataStatus.NACKED && isResourceCached + ? status + "_but_cached" : status; + } + @VisibleForTesting static final class MetricReporterCallback implements ResourceCallback, ServerConnectionCallback { diff --git a/xds/src/main/java/io/grpc/xds/client/ControlPlaneClient.java b/xds/src/main/java/io/grpc/xds/client/ControlPlaneClient.java index f33f54e441d..62076fb8bf1 100644 --- a/xds/src/main/java/io/grpc/xds/client/ControlPlaneClient.java +++ b/xds/src/main/java/io/grpc/xds/client/ControlPlaneClient.java @@ -234,13 +234,7 @@ void readyHandler() { */ // Must be synchronized boolean hasWorkingAdsStream() { - if (streamClosedNoResponse || shutdown) { - return false; - } - if (adsStream == null) { - return true; - } - return adsStream.responseReceived || !adsStream.closed; + return !streamClosedNoResponse; } diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClient.java b/xds/src/main/java/io/grpc/xds/client/XdsClient.java index 85aafa417c4..e3cef6a355f 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClient.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClient.java @@ -155,44 +155,46 @@ public static final class ResourceMetadata { private final String version; private final ResourceMetadataStatus status; private final long updateTimeNanos; + private final boolean cached; @Nullable private final Any rawResource; @Nullable private final UpdateFailureState errorState; private ResourceMetadata( - ResourceMetadataStatus status, String version, long updateTimeNanos, + ResourceMetadataStatus status, String version, long updateTimeNanos, boolean cached, @Nullable Any rawResource, @Nullable UpdateFailureState errorState) { this.status = checkNotNull(status, "status"); this.version = checkNotNull(version, "version"); this.updateTimeNanos = updateTimeNanos; + this.cached = cached; this.rawResource = rawResource; this.errorState = errorState; } - static ResourceMetadata newResourceMetadataUnknown() { - return new ResourceMetadata(ResourceMetadataStatus.UNKNOWN, "", 0, null, null); + public static ResourceMetadata newResourceMetadataUnknown() { + return new ResourceMetadata(ResourceMetadataStatus.UNKNOWN, "", 0, false,null, null); } - static ResourceMetadata newResourceMetadataRequested() { - return new ResourceMetadata(ResourceMetadataStatus.REQUESTED, "", 0, null, null); + public static ResourceMetadata newResourceMetadataRequested(boolean cached) { + return new ResourceMetadata(ResourceMetadataStatus.REQUESTED, "", 0, cached, null, null); } - static ResourceMetadata newResourceMetadataDoesNotExist() { - return new ResourceMetadata(ResourceMetadataStatus.DOES_NOT_EXIST, "", 0, null, null); + public static ResourceMetadata newResourceMetadataDoesNotExist() { + return new ResourceMetadata(ResourceMetadataStatus.DOES_NOT_EXIST, "", 0, false, null, null); } public static ResourceMetadata newResourceMetadataAcked( Any rawResource, String version, long updateTimeNanos) { checkNotNull(rawResource, "rawResource"); return new ResourceMetadata( - ResourceMetadataStatus.ACKED, version, updateTimeNanos, rawResource, null); + ResourceMetadataStatus.ACKED, version, updateTimeNanos, true, rawResource, null); } - static ResourceMetadata newResourceMetadataNacked( + public static ResourceMetadata newResourceMetadataNacked( ResourceMetadata metadata, String failedVersion, long failedUpdateTime, - String failedDetails) { + String failedDetails, boolean cached) { checkNotNull(metadata, "metadata"); return new ResourceMetadata(ResourceMetadataStatus.NACKED, - metadata.getVersion(), metadata.getUpdateTimeNanos(), metadata.getRawResource(), + metadata.getVersion(), metadata.getUpdateTimeNanos(), cached, metadata.getRawResource(), new UpdateFailureState(failedVersion, failedUpdateTime, failedDetails)); } @@ -211,6 +213,11 @@ public long getUpdateTimeNanos() { return updateTimeNanos; } + /** Returns whether the resource was cached. */ + public boolean isCached() { + return cached; + } + /** The last successfully updated xDS resource as it was returned by the server. */ @Nullable public Any getRawResource() { @@ -390,20 +397,6 @@ public interface ServerConnectionCallback { void reportServerConnectionGauge(boolean isConnected, String xdsServer); } - /** - * Reports the number of resources in each cache state. - * - *

Cache state is determined by two factors: - *

    - *
  • Whether the resource is cached. - *
  • The {@link io.grpc.xds.client.XdsClient.ResourceMetadata.ResourceMetadataStatus} of the - * resource. - *
- */ - public SettableFuture reportResourceCounts(ResourceCallback callback) { - throw new UnsupportedOperationException(); - } - /** * Reports whether xDS client has a working ADS stream to xDS server. * The definition of a working stream is defined in gRFC A78. diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java index cf47c046165..2c581f867d2 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java @@ -40,7 +40,6 @@ import io.grpc.internal.TimeProvider; import io.grpc.xds.client.Bootstrapper.AuthorityInfo; import io.grpc.xds.client.Bootstrapper.ServerInfo; -import io.grpc.xds.client.XdsClient.ResourceMetadata.ResourceMetadataStatus; import io.grpc.xds.client.XdsClient.ResourceStore; import io.grpc.xds.client.XdsLogger.XdsLogLevel; import java.net.URI; @@ -48,7 +47,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -542,32 +540,6 @@ public SettableFuture reportServerConnections(ServerConnectionCallback cal return future; } - @Override - public SettableFuture reportResourceCounts(ResourceCallback callback) { - SettableFuture future = SettableFuture.create(); - syncContext.execute(() -> { - for (XdsResourceType resourceType : resourceSubscribers.keySet()) { - Map resourceCountsByState = new HashMap<>(); - for (ResourceSubscriber subscriber : - resourceSubscribers.get(resourceType).values()) { - String cacheState = cacheStateFromResourceStatus(subscriber.metadata, - subscriber.data != null); - resourceCountsByState.compute(cacheState, (k, v) -> (v == null) ? 1 : v + 1); - } - resourceCountsByState.forEach((cacheState, count) -> - callback.reportResourceCountGauge(count, cacheState, resourceType.typeUrl())); - } - future.set(null); - }); - return future; - } - - private String cacheStateFromResourceStatus(ResourceMetadata metadata, boolean isResourceCached) { - String status = metadata.getStatus().toString().toLowerCase(Locale.ROOT); - return metadata.getStatus() == ResourceMetadataStatus.NACKED && isResourceCached - ? status + "_but_cached" : status; - } - /** Tracks a single subscribed resource. */ private final class ResourceSubscriber { @Nullable private final ServerInfo serverInfo; @@ -663,7 +635,7 @@ public String toString() { } // Initial fetch scheduled or rescheduled, transition metadata state to REQUESTED. - metadata = ResourceMetadata.newResourceMetadataRequested(); + metadata = ResourceMetadata.newResourceMetadataRequested(this.data != null); respTimer = syncContext.schedule( new ResourceNotFound(), INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS, @@ -706,10 +678,10 @@ void onData(ParsedResource parsedResource, String version, long updateTime, respTimer.cancel(); respTimer = null; } - this.metadata = ResourceMetadata - .newResourceMetadataAcked(parsedResource.getRawResource(), version, updateTime); ResourceUpdate oldData = this.data; this.data = parsedResource.getResourceUpdate(); + this.metadata = ResourceMetadata + .newResourceMetadataAcked(parsedResource.getRawResource(), version, updateTime); absent = false; if (resourceDeletionIgnored) { logger.log(XdsLogLevel.FORCE_INFO, "xds server {0}: server returned new version " @@ -803,7 +775,8 @@ void onError(Status error, @Nullable ProcessingTracker tracker) { void onRejected(String rejectedVersion, long rejectedTime, String rejectedDetails) { metadata = ResourceMetadata - .newResourceMetadataNacked(metadata, rejectedVersion, rejectedTime, rejectedDetails); + .newResourceMetadataNacked(metadata, rejectedVersion, rejectedTime, rejectedDetails, + data != null); } private void notifyWatcher(ResourceWatcher watcher, T update) { diff --git a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java index 61dd113e25c..555dfc10640 100644 --- a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java +++ b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java @@ -90,7 +90,6 @@ import io.grpc.xds.client.EnvoyProtoData.Node; import io.grpc.xds.client.LoadStatsManager2.ClusterDropStats; import io.grpc.xds.client.Locality; -import io.grpc.xds.client.XdsClient.ResourceCallback; import io.grpc.xds.client.XdsClient.ResourceMetadata; import io.grpc.xds.client.XdsClient.ResourceMetadata.ResourceMetadataStatus; import io.grpc.xds.client.XdsClient.ResourceMetadata.UpdateFailureState; @@ -108,7 +107,9 @@ import java.util.ArrayDeque; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Queue; import java.util.concurrent.BlockingDeque; @@ -295,8 +296,6 @@ public long currentTimeNanos() { @Mock private XdsClientMetricReporter xdsClientMetricReporter; @Mock - private ResourceCallback resourceCallback; - @Mock private ServerConnectionCallback serverConnectionCallback; private ManagedChannel channel; @@ -635,27 +634,28 @@ private void verifyServerFailureCount(int times, long serverFailureCount, String eq(xdsServer)); } - /** - * Invokes the callback, which will be called by {@link XdsClientMetricReporter} to record - * number of resources in each cacheState {@link ResourceMetadata#getStatus()}. - */ - private void callback_ReportResourceCount() { - try { - Future unused = xdsClient.reportResourceCounts(resourceCallback); - } catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw new AssertionError(e); + private void verifyResourceCountByCacheState(XdsResourceType type, String cacheState, + long resourceCount) { + Map metadataMap = awaitSubscribedResourcesMetadata().get(type); + assertThat(metadataMap).isNotNull(); + + // Calculate number of resources in cacheState + Map resourceCountsByState = new HashMap<>(); + for (ResourceMetadata metadata : metadataMap.values()) { + String cacheStateFromResourceStatus = cacheStateFromResourceStatus(metadata.getStatus(), + metadata.isCached()); + resourceCountsByState.compute(cacheStateFromResourceStatus, + (k, v) -> (v == null) ? 1 : v + 1); } + + assertThat(resourceCountsByState.get(cacheState)).isEqualTo(resourceCount); } - private void verifyResourceCountByCacheState(int times, long resourceCount, - String cacheState, String resourceTypeUrl) { - verify(resourceCallback, times(times)).reportResourceCountGauge( - eq(resourceCount), - eq(cacheState), - eq(resourceTypeUrl)); + private String cacheStateFromResourceStatus(ResourceMetadataStatus metadataStatus, + boolean isResourceCached) { + String status = metadataStatus.toString().toLowerCase(Locale.ROOT); + return metadataStatus == ResourceMetadataStatus.NACKED && isResourceCached + ? status + "_but_cached" : status; } /** @@ -693,9 +693,6 @@ public void ldsResourceNotFound() { verifyNoInteractions(ldsResourceWatcher); verifyResourceMetadataRequested(LDS, LDS_RESOURCE); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); // Server failed to return subscribed resource within expected time window. fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); verify(ldsResourceWatcher).onResourceDoesNotExist(LDS_RESOURCE); @@ -703,8 +700,6 @@ public void ldsResourceNotFound() { verifyResourceMetadataDoesNotExist(LDS, LDS_RESOURCE); // Check metric data. verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl()); } @Test @@ -781,9 +776,6 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataRequested(LDS, "B"); verifyResourceMetadataRequested(LDS, "C"); verifySubscribedResourcesMetadataSizes(3, 0, 0, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "requested", LDS.typeUrl()); // LDS -> {A, B, C}, version 1 ImmutableMap resourcesV1 = ImmutableMap.of( @@ -797,8 +789,6 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); // Check metric data. verifyResourceValidInvalidCount(1, 3, 0, xdsServerInfo.target(), LDS.typeUrl()); - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "acked", LDS.typeUrl()); call.verifyRequest(LDS, subscribedResourceNames, VERSION_1, "0000", NODE); // LDS -> {A, B}, version 2 @@ -818,18 +808,9 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceValidInvalidCount(1, 1, 1, xdsServerInfo.target(), LDS.typeUrl()); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(LDS, "C"); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl()); } else { // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl()); } call.verifyRequestNack(LDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); @@ -844,16 +825,9 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceValidInvalidCount(1, 2, 0, xdsServerInfo.target(), LDS.typeUrl()); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(LDS, "A"); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "does_not_exist", LDS.typeUrl()); - verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl()); } else { // When resource deletion is disabled, {A} stays ACKed in the previous version VERSION_2. verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 3, "acked", LDS.typeUrl()); } verifyResourceMetadataAcked(LDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); verifyResourceMetadataAcked(LDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); @@ -879,10 +853,6 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti verifyResourceMetadataRequested(RDS, "B.1"); verifyResourceMetadataRequested(RDS, "C.1"); verifySubscribedResourcesMetadataSizes(3, 0, 3, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "requested", LDS.typeUrl()); - verifyResourceCountByCacheState(1, 3, "requested", RDS.typeUrl()); // LDS -> {A, B, C}, version 1 ImmutableMap resourcesV1 = ImmutableMap.of( @@ -895,10 +865,6 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti verifyResourceMetadataAcked(LDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "acked", LDS.typeUrl()); - verifyResourceCountByCacheState(2, 3, "requested", RDS.typeUrl()); call.verifyRequest(LDS, subscribedResourceNames, VERSION_1, "0000", NODE); // RDS -> {A.1, B.1, C.1}, version 1 @@ -914,9 +880,6 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti verifyResourceMetadataAcked(RDS, "C.1", resourcesV11.get("C.1"), VERSION_1, TIME_INCREMENT * 2); // Check metric data. verifyResourceValidInvalidCount(1, 3, 0, xdsServerInfo.target(), RDS.typeUrl()); - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 3, "acked", LDS.typeUrl()); - verifyResourceCountByCacheState(1, 3, "acked", RDS.typeUrl()); // LDS -> {A, B}, version 2 // Failed to parse endpoint B @@ -936,20 +899,9 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti errorsV2); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(LDS, "C"); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl()); - verifyResourceCountByCacheState(2, 3, "acked", RDS.typeUrl()); } else { // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl()); - verifyResourceCountByCacheState(2, 3, "acked", RDS.typeUrl()); } call.verifyRequestNack(LDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); // {A.1} -> version 1 @@ -1015,9 +967,6 @@ public void wrappedLdsResource_preferWrappedName() { public void ldsResourceFound_containsRdsName() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); call.sendResponse(LDS, testListenerRds, VERSION_1, "0000"); // Client sends an ACK LDS request. @@ -1027,9 +976,6 @@ public void ldsResourceFound_containsRdsName() { assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerRds, VERSION_1, TIME_INCREMENT); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); } @Test @@ -1072,9 +1018,6 @@ public void ldsResourceUpdated() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); verifyResourceMetadataRequested(LDS, LDS_RESOURCE); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); // Initial LDS response. call.sendResponse(LDS, testListenerVhosts, VERSION_1, "0000"); @@ -1082,9 +1025,6 @@ public void ldsResourceUpdated() { verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); verifyGoldenListenerVhosts(ldsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); // Updated LDS response. call.sendResponse(LDS, testListenerRds, VERSION_2, "0001"); @@ -1093,9 +1033,6 @@ public void ldsResourceUpdated() { verifyGoldenListenerRds(ldsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerRds, VERSION_2, TIME_INCREMENT * 2); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl()); assertThat(channelForCustomAuthority).isNull(); assertThat(channelForEmptyAuthority).isNull(); } @@ -1105,9 +1042,6 @@ public void cancelResourceWatcherNotRemoveUrlSubscribers() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); verifyResourceMetadataRequested(LDS, LDS_RESOURCE); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); // Initial LDS response. call.sendResponse(LDS, testListenerVhosts, VERSION_1, "0000"); @@ -1115,9 +1049,6 @@ public void cancelResourceWatcherNotRemoveUrlSubscribers() { verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); verifyGoldenListenerVhosts(ldsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); xdsClient.watchXdsResource(XdsListenerResource.getInstance(), LDS_RESOURCE + "1", ldsResourceWatcher); @@ -1133,9 +1064,6 @@ public void cancelResourceWatcherNotRemoveUrlSubscribers() { verifyGoldenListenerVhosts(ldsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts2, VERSION_2, TIME_INCREMENT * 2); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl()); } @Test @@ -1200,9 +1128,6 @@ public void ldsResourceUpdated_withXdstpResourceName_withWrongType() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), ldsResourceName, ldsResourceWatcher); assertThat(channelForCustomAuthority).isNotNull(); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); String ldsResourceNameWithWrongType = "xdstp://authority.xds.com/envoy.config.route.v3.RouteConfiguration/listener1"; @@ -1214,9 +1139,6 @@ public void ldsResourceUpdated_withXdstpResourceName_withWrongType() { LDS, ldsResourceName, "", "0000", NODE, ImmutableList.of( "Unsupported resource name: " + ldsResourceNameWithWrongType + " for type: LDS")); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "requested", LDS.typeUrl()); } @Test @@ -1334,9 +1256,6 @@ public void edsResourceUpdated_withXdstpResourceName_unknownAuthority() { public void ldsResourceUpdate_withFaultInjection() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); Any listener = Any.pack( mf.buildListenerWithApiListener( LDS_RESOURCE, @@ -1375,9 +1294,6 @@ public void ldsResourceUpdate_withFaultInjection() { verify(ldsResourceWatcher).onChanged(ldsUpdateCaptor.capture()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, listener, VERSION_1, TIME_INCREMENT); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); LdsUpdate ldsUpdate = ldsUpdateCaptor.getValue(); assertThat(ldsUpdate.httpConnectionManager().virtualHosts()).hasSize(2); @@ -1408,9 +1324,6 @@ public void ldsResourceDeleted() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); verifyResourceMetadataRequested(LDS, LDS_RESOURCE); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); // Initial LDS response. call.sendResponse(LDS, testListenerVhosts, VERSION_1, "0000"); @@ -1419,9 +1332,6 @@ public void ldsResourceDeleted() { verifyGoldenListenerVhosts(ldsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); // Empty LDS response deletes the listener. call.sendResponse(LDS, Collections.emptyList(), VERSION_2, "0001"); @@ -1429,9 +1339,6 @@ public void ldsResourceDeleted() { verify(ldsResourceWatcher).onResourceDoesNotExist(LDS_RESOURCE); verifyResourceMetadataDoesNotExist(LDS, LDS_RESOURCE); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl()); } /** @@ -1445,9 +1352,6 @@ public void ldsResourceDeleted_ignoreResourceDeletion() { DiscoveryRpcCall call = startResourceWatcher(XdsListenerResource.getInstance(), LDS_RESOURCE, ldsResourceWatcher); verifyResourceMetadataRequested(LDS, LDS_RESOURCE); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); // Initial LDS response. call.sendResponse(LDS, testListenerVhosts, VERSION_1, "0000"); @@ -1456,9 +1360,6 @@ public void ldsResourceDeleted_ignoreResourceDeletion() { verifyGoldenListenerVhosts(ldsUpdateCaptor.getValue()); verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); // Empty LDS response does not delete the listener. call.sendResponse(LDS, Collections.emptyList(), VERSION_2, "0001"); @@ -1466,9 +1367,6 @@ public void ldsResourceDeleted_ignoreResourceDeletion() { // The resource is still ACKED at VERSION_1 (no changes). verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl()); // onResourceDoesNotExist not called verify(ldsResourceWatcher, never()).onResourceDoesNotExist(LDS_RESOURCE); @@ -1481,9 +1379,6 @@ public void ldsResourceDeleted_ignoreResourceDeletion() { verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_3, TIME_INCREMENT * 3); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(3, 1, "acked", LDS.typeUrl()); verifyNoMoreInteractions(ldsResourceWatcher); } @@ -1502,9 +1397,6 @@ public void multipleLdsWatchers() { verifyResourceMetadataRequested(LDS, LDS_RESOURCE); verifyResourceMetadataRequested(LDS, ldsResourceTwo); verifySubscribedResourcesMetadataSizes(2, 0, 0, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "requested", LDS.typeUrl()); fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); verify(ldsResourceWatcher).onResourceDoesNotExist(LDS_RESOURCE); @@ -1513,9 +1405,6 @@ public void multipleLdsWatchers() { verifyResourceMetadataDoesNotExist(LDS, LDS_RESOURCE); verifyResourceMetadataDoesNotExist(LDS, ldsResourceTwo); verifySubscribedResourcesMetadataSizes(2, 0, 0, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "does_not_exist", LDS.typeUrl()); Any listenerTwo = Any.pack(mf.buildListenerWithApiListenerForRds(ldsResourceTwo, RDS_RESOURCE)); call.sendResponse(LDS, ImmutableList.of(testListenerVhosts, listenerTwo), VERSION_1, "0000"); @@ -1534,9 +1423,6 @@ public void multipleLdsWatchers() { verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerVhosts, VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, ldsResourceTwo, listenerTwo, VERSION_1, TIME_INCREMENT); verifySubscribedResourcesMetadataSizes(2, 0, 0, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl()); } @Test @@ -1603,9 +1489,6 @@ public void rdsResponseErrorHandling_nackWeightedSumZero() { DiscoveryRpcCall call = startResourceWatcher(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, rdsResourceWatcher); verifyResourceMetadataRequested(RDS, RDS_RESOURCE); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl()); io.envoyproxy.envoy.config.route.v3.RouteAction routeAction = io.envoyproxy.envoy.config.route.v3.RouteAction.newBuilder() @@ -1641,9 +1524,6 @@ public void rdsResponseErrorHandling_nackWeightedSumZero() { + "contains invalid route : Route [route-blade] contains invalid RouteAction: " + "Sum of cluster weights should be above 0."); verifySubscribedResourcesMetadataSizes(0, 0, 1, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "nacked", RDS.typeUrl()); // The response is NACKed with the same error message. call.verifyRequestNack(RDS, RDS_RESOURCE, "", "0000", NODE, errors); verify(rdsResourceWatcher, never()).onChanged(any(RdsUpdate.class)); @@ -1668,9 +1548,6 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataRequested(RDS, "B"); verifyResourceMetadataRequested(RDS, "C"); verifySubscribedResourcesMetadataSizes(0, 0, 3, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "requested", RDS.typeUrl()); // RDS -> {A, B, C}, version 1 List vhostsV1 = mf.buildOpaqueVirtualHosts(1); @@ -1685,8 +1562,6 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataAcked(RDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); // Check metric data. verifyResourceValidInvalidCount(1, 3, 0, xdsServerInfo.target(), RDS.typeUrl()); - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "acked", RDS.typeUrl()); call.verifyRequest(RDS, subscribedResourceNames, VERSION_1, "0000", NODE); // RDS -> {A, B}, version 2 @@ -1706,10 +1581,6 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataNacked(RDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, VERSION_2, TIME_INCREMENT * 2, errorsV2); verifyResourceMetadataAcked(RDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", RDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", RDS.typeUrl()); call.verifyRequestNack(RDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); // RDS -> {B, C} version 3 @@ -1726,9 +1597,6 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataAcked(RDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); verifyResourceMetadataAcked(RDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); call.verifyRequest(RDS, subscribedResourceNames, VERSION_3, "0002", NODE); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 3, "acked", RDS.typeUrl()); verifySubscribedResourcesMetadataSizes(0, 0, 3, 0); } @@ -1802,9 +1670,6 @@ public void rdsResourceUpdated() { DiscoveryRpcCall call = startResourceWatcher(XdsRouteConfigureResource.getInstance(), RDS_RESOURCE, rdsResourceWatcher); verifyResourceMetadataRequested(RDS, RDS_RESOURCE); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl()); // Initial RDS response. call.sendResponse(RDS, testRouteConfig, VERSION_1, "0000"); @@ -1812,9 +1677,6 @@ public void rdsResourceUpdated() { verify(rdsResourceWatcher).onChanged(rdsUpdateCaptor.capture()); verifyGoldenRouteConfig(rdsUpdateCaptor.getValue()); verifyResourceMetadataAcked(RDS, RDS_RESOURCE, testRouteConfig, VERSION_1, TIME_INCREMENT); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", RDS.typeUrl()); // Updated RDS response. Any routeConfigUpdated = @@ -1827,9 +1689,6 @@ public void rdsResourceUpdated() { assertThat(rdsUpdateCaptor.getValue().virtualHosts).hasSize(4); verifyResourceMetadataAcked(RDS, RDS_RESOURCE, routeConfigUpdated, VERSION_2, TIME_INCREMENT * 2); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "acked", RDS.typeUrl()); } @Test @@ -1841,10 +1700,6 @@ public void rdsResourceDeletedByLdsApiListener() { verifyResourceMetadataRequested(LDS, LDS_RESOURCE); verifyResourceMetadataRequested(RDS, RDS_RESOURCE); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl()); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); call.sendResponse(LDS, testListenerRds, VERSION_1, "0000"); @@ -1853,10 +1708,6 @@ public void rdsResourceDeletedByLdsApiListener() { verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerRds, VERSION_1, TIME_INCREMENT); verifyResourceMetadataRequested(RDS, RDS_RESOURCE); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); - verifyResourceCountByCacheState(2, 1, "requested", RDS.typeUrl()); call.sendResponse(RDS, testRouteConfig, VERSION_1, "0000"); verify(rdsResourceWatcher).onChanged(rdsUpdateCaptor.capture()); @@ -1864,10 +1715,6 @@ public void rdsResourceDeletedByLdsApiListener() { verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerRds, VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(RDS, RDS_RESOURCE, testRouteConfig, VERSION_1, TIME_INCREMENT * 2); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "acked", RDS.typeUrl()); // The Listener is getting replaced configured with an RDS name, to the one configured with // vhosts. Expect the RDS resources to be discarded. @@ -1881,10 +1728,6 @@ public void rdsResourceDeletedByLdsApiListener() { verifyResourceMetadataAcked( LDS, LDS_RESOURCE, testListenerVhosts, VERSION_2, TIME_INCREMENT * 3); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(3, 1, "acked", LDS.typeUrl()); - verifyResourceCountByCacheState(2, 1, "acked", RDS.typeUrl()); } @Test @@ -1896,10 +1739,6 @@ public void rdsResourcesDeletedByLdsTcpListener() { verifyResourceMetadataRequested(LDS, LISTENER_RESOURCE); verifyResourceMetadataRequested(RDS, RDS_RESOURCE); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl()); Message hcmFilter = mf.buildHttpConnectionManagerFilter( RDS_RESOURCE, null, Collections.singletonList(mf.buildTerminalFilter())); @@ -1924,20 +1763,12 @@ public void rdsResourcesDeletedByLdsTcpListener() { verifyResourceMetadataAcked(LDS, LISTENER_RESOURCE, packedListener, VERSION_1, TIME_INCREMENT); verifyResourceMetadataRequested(RDS, RDS_RESOURCE); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); - verifyResourceCountByCacheState(2, 1, "requested", RDS.typeUrl()); // Simulates receiving the requested RDS resource. call.sendResponse(RDS, testRouteConfig, VERSION_1, "0000"); verify(rdsResourceWatcher).onChanged(rdsUpdateCaptor.capture()); verifyGoldenRouteConfig(rdsUpdateCaptor.getValue()); verifyResourceMetadataAcked(RDS, RDS_RESOURCE, testRouteConfig, VERSION_1, TIME_INCREMENT * 2); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "acked", RDS.typeUrl()); // Simulates receiving an updated version of the requested LDS resource as a TCP listener // with a filter chain containing inlined RouteConfiguration. @@ -1964,10 +1795,6 @@ public void rdsResourcesDeletedByLdsTcpListener() { verifyResourceMetadataAcked( LDS, LISTENER_RESOURCE, packedListener, VERSION_2, TIME_INCREMENT * 3); verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(3, 1, "acked", LDS.typeUrl()); - verifyResourceCountByCacheState(2, 1, "acked", RDS.typeUrl()); } @Test @@ -2099,9 +1926,6 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataRequested(CDS, "B"); verifyResourceMetadataRequested(CDS, "C"); verifySubscribedResourcesMetadataSizes(0, 3, 0, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "requested", CDS.typeUrl()); // CDS -> {A, B, C}, version 1 ImmutableMap resourcesV1 = ImmutableMap.of( @@ -2121,9 +1945,6 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataAcked(CDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(CDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(CDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "acked", CDS.typeUrl()); call.verifyRequest(CDS, subscribedResourceNames, VERSION_1, "0000", NODE); // CDS -> {A, B}, version 2 @@ -2144,19 +1965,10 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataNacked(CDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, VERSION_2, TIME_INCREMENT * 2, errorsV2); if (!ignoreResourceDeletion()) { - verifyResourceMetadataDoesNotExist(CDS, "C"); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", CDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", CDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "does_not_exist", CDS.typeUrl()); + verifyResourceMetadataDoesNotExist(CDS, "C");; } else { // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. verifyResourceMetadataAcked(CDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", CDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", CDS.typeUrl()); } call.verifyRequestNack(CDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); @@ -2175,16 +1987,9 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { CDS.typeUrl()); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(CDS, "A"); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", CDS.typeUrl()); - verifyResourceCountByCacheState(2, 1, "does_not_exist", CDS.typeUrl()); } else { // When resource deletion is disabled, {A} stays ACKed in the previous version VERSION_2. verifyResourceMetadataAcked(CDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 3, "acked", CDS.typeUrl()); } verifyResourceMetadataAcked(CDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); verifyResourceMetadataAcked(CDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); @@ -2210,10 +2015,6 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti verifyResourceMetadataRequested(EDS, "B.1"); verifyResourceMetadataRequested(EDS, "C.1"); verifySubscribedResourcesMetadataSizes(0, 3, 0, 3); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "requested", CDS.typeUrl()); - verifyResourceCountByCacheState(1, 3, "requested", EDS.typeUrl()); // CDS -> {A, B, C}, version 1 ImmutableMap resourcesV1 = ImmutableMap.of( @@ -2233,10 +2034,6 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti verifyResourceMetadataAcked(CDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(CDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(CDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "acked", CDS.typeUrl()); - verifyResourceCountByCacheState(2, 3, "requested", EDS.typeUrl()); call.verifyRequest(CDS, subscribedResourceNames, VERSION_1, "0000", NODE); // EDS -> {A.1, B.1, C.1}, version 1 @@ -2253,10 +2050,6 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti verifyResourceMetadataAcked(EDS, "A.1", resourcesV11.get("A.1"), VERSION_1, TIME_INCREMENT * 2); verifyResourceMetadataAcked(EDS, "B.1", resourcesV11.get("B.1"), VERSION_1, TIME_INCREMENT * 2); verifyResourceMetadataAcked(EDS, "C.1", resourcesV11.get("C.1"), VERSION_1, TIME_INCREMENT * 2); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 3, "acked", CDS.typeUrl()); - verifyResourceCountByCacheState(1, 3, "acked", EDS.typeUrl()); // CDS -> {A, B}, version 2 // Failed to parse endpoint B @@ -2278,20 +2071,9 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti errorsV2); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(CDS, "C"); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", CDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", CDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "does_not_exist", CDS.typeUrl()); - verifyResourceCountByCacheState(2, 3, "acked", EDS.typeUrl()); } else { // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. verifyResourceMetadataAcked(CDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", CDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", CDS.typeUrl()); - verifyResourceCountByCacheState(2, 3, "acked", EDS.typeUrl()); } call.verifyRequestNack(CDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); // {A.1} -> version 1 @@ -2535,9 +2317,6 @@ public void cdsResponseWithNewUpstreamTlsContext() { public void cdsResponseErrorHandling_badUpstreamTlsContext() { DiscoveryRpcCall call = startResourceWatcher(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl()); // Management server sends back CDS response with UpstreamTlsContext. List clusters = ImmutableList.of(Any @@ -2555,9 +2334,6 @@ public void cdsResponseErrorHandling_badUpstreamTlsContext() { call.verifyRequestNack(CDS, CDS_RESOURCE, "", "0000", NODE, ImmutableList.of(errorMsg)); verify(cdsResourceWatcher).onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, errorMsg); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "requested", CDS.typeUrl()); } /** @@ -2568,9 +2344,6 @@ public void cdsResponseErrorHandling_badUpstreamTlsContext() { public void cdsResponseWithOutlierDetection() { DiscoveryRpcCall call = startResourceWatcher(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl()); OutlierDetection outlierDetectionXds = OutlierDetection.newBuilder() .setInterval(Durations.fromNanos(100)) @@ -2630,9 +2403,6 @@ public void cdsResponseWithOutlierDetection() { verifyResourceMetadataAcked(CDS, CDS_RESOURCE, clusterEds, VERSION_1, TIME_INCREMENT); verifySubscribedResourcesMetadataSizes(0, 1, 0, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", CDS.typeUrl()); } /** @@ -2644,9 +2414,6 @@ public void cdsResponseWithInvalidOutlierDetectionNacks() { DiscoveryRpcCall call = startResourceWatcher(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl()); OutlierDetection outlierDetectionXds = OutlierDetection.newBuilder() .setMaxEjectionPercent(UInt32Value.of(101)).build(); @@ -2752,9 +2519,6 @@ public void validateOutlierDetection_enforcingFailurePercentageTooHigh() public void cdsResponseErrorHandling_badTransportSocketName() { DiscoveryRpcCall call = startResourceWatcher(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl()); // Management server sends back CDS response with UpstreamTlsContext. List clusters = ImmutableList.of(Any @@ -2768,9 +2532,6 @@ public void cdsResponseErrorHandling_badTransportSocketName() { String errorMsg = "CDS response Cluster 'cluster.googleapis.com' validation error: " + "transport-socket with name envoy.transport_sockets.bad not supported."; call.verifyRequestNack(CDS, CDS_RESOURCE, "", "0000", NODE, ImmutableList.of(errorMsg)); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "requested", CDS.typeUrl()); verify(cdsResourceWatcher).onError(errorCaptor.capture()); verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, errorMsg); } @@ -2828,9 +2589,6 @@ public void cachedCdsResource_data() { public void cachedCdsResource_absent() { DiscoveryRpcCall call = startResourceWatcher(XdsClusterResource.getInstance(), CDS_RESOURCE, cdsResourceWatcher); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl()); fakeClock.forwardTime(XdsClientImpl.INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); verify(cdsResourceWatcher).onResourceDoesNotExist(CDS_RESOURCE); ResourceWatcher watcher = mock(ResourceWatcher.class); @@ -2839,9 +2597,6 @@ public void cachedCdsResource_absent() { call.verifyNoMoreRequest(); verifyResourceMetadataDoesNotExist(CDS, CDS_RESOURCE); verifySubscribedResourcesMetadataSizes(0, 1, 0, 0); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "does_not_exist", CDS.typeUrl()); } @Test @@ -3123,9 +2878,6 @@ public void edsCleanupNonceAfterUnsubscription() { DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); assertThat(call).isNotNull(); call.verifyRequest(EDS, "A.1", "", "", NODE); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", EDS.typeUrl()); // EDS -> {A.1}, version 1 List dropOverloads = ImmutableList.of(); @@ -3136,9 +2888,6 @@ public void edsCleanupNonceAfterUnsubscription() { // {A.1} -> ACK, version 1 call.verifyRequest(EDS, "A.1", VERSION_1, "0000", NODE); verify(edsResourceWatcher, times(1)).onChanged(any()); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", EDS.typeUrl()); // trigger an EDS resource unsubscription. xdsClient.cancelXdsResourceWatch(XdsEndpointResource.getInstance(), "A.1", edsResourceWatcher); @@ -3149,9 +2898,6 @@ public void edsCleanupNonceAfterUnsubscription() { // same as the initial request xdsClient.watchXdsResource(XdsEndpointResource.getInstance(), "A.1", edsResourceWatcher); call.verifyRequest(EDS, "A.1", "", "", NODE, Mockito.timeout(2000).times(2)); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "requested", EDS.typeUrl()); } @Test @@ -3213,9 +2959,6 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataRequested(EDS, "A"); verifyResourceMetadataRequested(EDS, "B"); verifyResourceMetadataRequested(EDS, "C"); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "requested", EDS.typeUrl()); // EDS -> {A, B, C}, version 1 List dropOverloads = ImmutableList.of(mf.buildDropOverload("lb", 200)); @@ -3230,9 +2973,6 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataAcked(EDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(EDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(EDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "acked", EDS.typeUrl()); call.verifyRequest(EDS, subscribedResourceNames, VERSION_1, "0000", NODE); // EDS -> {A, B}, version 2 @@ -3253,10 +2993,6 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataNacked(EDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, VERSION_2, TIME_INCREMENT * 2, errorsV2); verifyResourceMetadataAcked(EDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", EDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", EDS.typeUrl()); call.verifyRequestNack(EDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); // EDS -> {B, C} version 3 @@ -3275,9 +3011,6 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { verifyResourceMetadataAcked(EDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); call.verifyRequest(EDS, subscribedResourceNames, VERSION_3, "0002", NODE); verifySubscribedResourcesMetadataSizes(0, 0, 0, 3); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 3, "acked", EDS.typeUrl()); } @Test @@ -3357,9 +3090,6 @@ public void flowControlAbsent() throws Exception { anotherWatcher, fakeWatchClock.getScheduledExecutorService()); verifyResourceMetadataRequested(CDS, CDS_RESOURCE); verifyResourceMetadataRequested(CDS, anotherCdsResource); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "requested", CDS.typeUrl()); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); call.verifyRequest(CDS, Arrays.asList(CDS_RESOURCE, anotherCdsResource), "", "", NODE); @@ -3367,10 +3097,6 @@ public void flowControlAbsent() throws Exception { call.sendResponse(CDS, testClusterRoundRobin, VERSION_1, "0000"); verifyResourceMetadataAcked( CDS, CDS_RESOURCE, testClusterRoundRobin, VERSION_1, TIME_INCREMENT); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", CDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl()); call.verifyRequest(CDS, Arrays.asList(CDS_RESOURCE, anotherCdsResource), VERSION_1, "0000", NODE); verifyNoInteractions(cdsResourceWatcher, anotherWatcher); @@ -3398,10 +3124,6 @@ public void flowControlAbsent() throws Exception { assertThat(call.isReady()).isFalse(); verifyResourceMetadataAcked( CDS, CDS_RESOURCE, testClusterRoundRobin, VERSION_1, TIME_INCREMENT); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "acked", CDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "does_not_exist", CDS.typeUrl()); barrier.await(); verify(cdsResourceWatcher, atLeastOnce()).onChanged(any()); String errorMsg = "CDS response Cluster 'cluster.googleapis.com2' validation error: " @@ -3413,10 +3135,6 @@ public void flowControlAbsent() throws Exception { verify(cdsResourceWatcher, times(2)).onChanged(any()); verify(anotherWatcher).onResourceDoesNotExist(eq(anotherCdsResource)); verify(anotherWatcher).onError(any()); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(3, 1, "acked", CDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "nacked", CDS.typeUrl()); } private Answer blockUpdate(CyclicBarrier barrier) { @@ -3566,10 +3284,6 @@ public void edsResourceDeletedByCds() { verifyResourceMetadataRequested(EDS, EDS_RESOURCE); verifyResourceMetadataRequested(EDS, resource); verifySubscribedResourcesMetadataSizes(0, 2, 0, 2); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "requested", CDS.typeUrl()); - verifyResourceCountByCacheState(1, 2, "requested", EDS.typeUrl()); DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); List clusters = ImmutableList.of( @@ -3591,10 +3305,6 @@ public void edsResourceDeletedByCds() { verifyResourceMetadataAcked(CDS, CDS_RESOURCE, clusters.get(1), VERSION_1, TIME_INCREMENT); verifyResourceMetadataRequested(EDS, EDS_RESOURCE); verifyResourceMetadataRequested(EDS, resource); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", CDS.typeUrl()); - verifyResourceCountByCacheState(2, 2, "requested", EDS.typeUrl()); List clusterLoadAssignments = ImmutableList.of( @@ -3625,10 +3335,6 @@ public void edsResourceDeletedByCds() { verifyResourceMetadataAcked(CDS, resource, clusters.get(0), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(CDS, CDS_RESOURCE, clusters.get(1), VERSION_1, TIME_INCREMENT); verifySubscribedResourcesMetadataSizes(0, 2, 0, 2); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 2, "acked", CDS.typeUrl()); - verifyResourceCountByCacheState(1, 2, "acked", EDS.typeUrl()); clusters = ImmutableList.of( Any.pack(mf.buildEdsCluster(resource, null, "round_robin", null, null, true, null, @@ -3650,10 +3356,6 @@ public void edsResourceDeletedByCds() { EDS, resource, clusterLoadAssignments.get(1), VERSION_1, TIME_INCREMENT * 2); // no change verifyResourceMetadataAcked(CDS, resource, clusters.get(0), VERSION_2, TIME_INCREMENT * 3); verifyResourceMetadataAcked(CDS, CDS_RESOURCE, clusters.get(1), VERSION_2, TIME_INCREMENT * 3); - // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(3, 2, "acked", CDS.typeUrl()); - verifyResourceCountByCacheState(2, 2, "acked", EDS.typeUrl()); } @Test @@ -4013,11 +3715,6 @@ public void streamClosedAndRetryRestartsResourceInitialFetchTimerForUnresolvedRe // Check metric data. callback_ReportServerConnection(); verifyServerConnection(1, true, xdsServerInfo.target()); - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "requested", RDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "requested", EDS.typeUrl()); ScheduledTask ldsResourceTimeout = Iterables.getOnlyElement(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)); ScheduledTask rdsResourceTimeout = @@ -4031,22 +3728,12 @@ public void streamClosedAndRetryRestartsResourceInitialFetchTimerForUnresolvedRe // Check metric data. callback_ReportServerConnection(); verifyServerConnection(2, true, xdsServerInfo.target()); - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); - verifyResourceCountByCacheState(2, 1, "requested", RDS.typeUrl()); - verifyResourceCountByCacheState(2, 1, "requested", CDS.typeUrl()); - verifyResourceCountByCacheState(2, 1, "requested", EDS.typeUrl()); call.sendResponse(RDS, testRouteConfig, VERSION_1, "0000"); assertThat(rdsResourceTimeout.isCancelled()).isTrue(); // Check metric data. callback_ReportServerConnection(); verifyServerConnection(3, true, xdsServerInfo.target()); - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "acked", LDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "acked", RDS.typeUrl()); - verifyResourceCountByCacheState(3, 1, "requested", CDS.typeUrl()); - verifyResourceCountByCacheState(3, 1, "requested", EDS.typeUrl()); call.sendError(Status.UNAVAILABLE.asException()); assertThat(cdsResourceTimeout.isCancelled()).isTrue(); @@ -4058,11 +3745,6 @@ public void streamClosedAndRetryRestartsResourceInitialFetchTimerForUnresolvedRe // Check metric data. callback_ReportServerConnection(); verifyServerConnection(4, true, xdsServerInfo.target()); - callback_ReportResourceCount(); - verifyResourceCountByCacheState(3, 1, "acked", LDS.typeUrl()); - verifyResourceCountByCacheState(2, 1, "acked", RDS.typeUrl()); - verifyResourceCountByCacheState(4, 1, "requested", CDS.typeUrl()); - verifyResourceCountByCacheState(4, 1, "requested", EDS.typeUrl()); fakeClock.forwardNanos(10L); assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).hasSize(0); @@ -4227,9 +3909,6 @@ public void sendingToStoppedServer() throws Exception { // Check metric data. callback_ReportServerConnection(); verifyServerConnection(1, false, xdsServerInfo.target()); - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "unknown", LDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "requested", CDS.typeUrl()); // Restart the server xdsServer = cleanupRule.register( @@ -4247,9 +3926,6 @@ public void sendingToStoppedServer() throws Exception { // Check metric data. callback_ReportServerConnection(); verifyServerConnection(2, false, xdsServerInfo.target()); - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 1, "unknown", LDS.typeUrl()); - verifyResourceCountByCacheState(2, 1, "requested", CDS.typeUrl()); if (call == null) { // The first rpcRetry may have happened before the channel was ready fakeClock.forwardTime(50, TimeUnit.SECONDS); call = resourceDiscoveryCalls.poll(3, TimeUnit.SECONDS); @@ -4258,9 +3934,6 @@ public void sendingToStoppedServer() throws Exception { // Check metric data. callback_ReportServerConnection(); verifyServerConnection(3, false, xdsServerInfo.target()); - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "requested", LDS.typeUrl()); - verifyResourceCountByCacheState(3, 1, "requested", CDS.typeUrl()); // NOTE: There is a ScheduledExecutorService that may get involved due to the reconnect // so you cannot rely on the logic being single threaded. The timeout() in verifyRequest @@ -4276,9 +3949,6 @@ public void sendingToStoppedServer() throws Exception { // Check metric data. callback_ReportServerConnection(); verifyServerConnection(1, true, xdsServerInfo.target()); - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); - verifyResourceCountByCacheState(4, 1, "requested", CDS.typeUrl()); } catch (Throwable t) { throw t; // This allows putting a breakpoint here for debugging } @@ -4457,7 +4127,7 @@ public void serverFailureMetricReport_forRetryAndBackoff() { @Test - public void testReportResourceCounts() { + public void testResourceCounts() { List subscribedResourceNames = ImmutableList.of("A", "B", "C"); xdsClient.watchXdsResource(XdsListenerResource.getInstance(), "A", ldsResourceWatcher); xdsClient.watchXdsResource(XdsListenerResource.getInstance(), "B", ldsResourceWatcher); @@ -4469,8 +4139,7 @@ public void testReportResourceCounts() { verifyResourceMetadataRequested(LDS, "C"); verifySubscribedResourcesMetadataSizes(3, 0, 0, 0); // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "requested", LDS.typeUrl()); + verifyResourceCountByCacheState(LDS,"requested", 3); // LDS -> {A, B, C}, version 1 ImmutableMap resourcesV1 = ImmutableMap.of( @@ -4484,8 +4153,7 @@ public void testReportResourceCounts() { verifyResourceMetadataAcked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 3, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(LDS, "acked",3); call.verifyRequest(LDS, subscribedResourceNames, VERSION_1, "0000", NODE); // LDS -> {A, B}, version 2 @@ -4505,17 +4173,15 @@ public void testReportResourceCounts() { if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(LDS, "C"); // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 1, "acked", LDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "does_not_exist", LDS.typeUrl()); + verifyResourceCountByCacheState(LDS, "acked", 1); + verifyResourceCountByCacheState(LDS, "nacked_but_cached", 1); + verifyResourceCountByCacheState(LDS, "does_not_exist", 1); } else { // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl()); - verifyResourceCountByCacheState(1, 1, "nacked_but_cached", LDS.typeUrl()); + verifyResourceCountByCacheState(LDS, "acked", 2); + verifyResourceCountByCacheState(LDS, "nacked_but_cached", 1); } call.verifyRequestNack(LDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); @@ -4530,15 +4196,13 @@ public void testReportResourceCounts() { if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(LDS, "A"); // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(1, 2, "acked", LDS.typeUrl()); - verifyResourceCountByCacheState(2, 1, "does_not_exist", LDS.typeUrl()); + verifyResourceCountByCacheState(LDS, "acked", 2); + verifyResourceCountByCacheState(LDS, "does_not_exist", 1); } else { // When resource deletion is disabled, {A} stays ACKed in the previous version VERSION_2. verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); // Check metric data. - callback_ReportResourceCount(); - verifyResourceCountByCacheState(2, 3, "acked", LDS.typeUrl()); + verifyResourceCountByCacheState(LDS, "acked", 3); } verifyResourceMetadataAcked(LDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); verifyResourceMetadataAcked(LDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); diff --git a/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java b/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java index 4fb77f0be42..a15221a7464 100644 --- a/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java @@ -132,7 +132,7 @@ public void refCountedXdsClientObjectPool_getObjectCreatesNewInstanceIfAlreadySh provider.new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET, metricRecorder); XdsClient xdsClient1 = xdsClientPool.getObject(); assertThat(xdsClientPool.returnObject(xdsClient1)).isNull(); - assertThat(xdsClient1.isShutDown()).isTrue(); + assertThat(xdsClient1.isShutDogit wn()).isTrue(); XdsClient xdsClient2 = xdsClientPool.getObject(); assertThat(xdsClient2).isNotSameInstanceAs(xdsClient1); diff --git a/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java b/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java index ef4f2238ded..0723e958334 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java @@ -24,10 +24,17 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; +import com.google.protobuf.Any; +import io.envoyproxy.envoy.config.listener.v3.Listener; import io.grpc.MetricInstrument; import io.grpc.MetricRecorder; import io.grpc.MetricRecorder.BatchCallback; @@ -35,12 +42,15 @@ import io.grpc.MetricSink; import io.grpc.xds.XdsClientMetricReporterImpl.MetricReporterCallback; import io.grpc.xds.client.XdsClient; -import io.grpc.xds.client.XdsClient.ResourceCallback; +import io.grpc.xds.client.XdsClient.ResourceMetadata; import io.grpc.xds.client.XdsClient.ServerConnectionCallback; +import io.grpc.xds.client.XdsResourceType; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.logging.Handler; import java.util.logging.Level; import java.util.logging.LogRecord; @@ -116,8 +126,8 @@ public void reportServerFailure() { public void setXdsClient_reportMetrics() throws Exception { SettableFuture future = SettableFuture.create(); future.set(null); - when(mockXdsClient.reportResourceCounts(any(ResourceCallback.class))) - .thenReturn(future); + when(mockXdsClient.getSubscribedResourcesMetadataSnapshot()).thenReturn(Futures.immediateFuture( + ImmutableMap.of())); when(mockXdsClient.reportServerConnections(any(ServerConnectionCallback.class))) .thenReturn(future); reporter.setXdsClient(mockXdsClient); @@ -125,49 +135,38 @@ public void setXdsClient_reportMetrics() throws Exception { eqMetricInstrumentName("grpc.xds_client.connected"), eqMetricInstrumentName("grpc.xds_client.resources")); gaugeBatchCallbackCaptor.getValue().accept(mockBatchRecorder); - verify(mockXdsClient).reportResourceCounts(any(ResourceCallback.class)); verify(mockXdsClient).reportServerConnections(any(ServerConnectionCallback.class)); } @Test public void setXdsClient_reportCallbackMetrics_resourceCountsFails() { - List logs = new ArrayList<>(); - Handler testLogHandler = new Handler() { - @Override - public void publish(LogRecord record) { - logs.add(record); - } - - @Override - public void close() {} - - @Override - public void flush() {} - }; + TestlogHandler testLogHandler = new TestlogHandler(); Logger logger = Logger.getLogger(XdsClientMetricReporterImpl.class.getName()); logger.addHandler(testLogHandler); - // Create a future that will throw an exception - SettableFuture resourceCountsFuture = SettableFuture.create(); - resourceCountsFuture.setException(new Exception("test")); - when(mockXdsClient.reportResourceCounts(any())).thenReturn(resourceCountsFuture); + // For reporting resource counts connections, return a normally completed future + SettableFuture future = SettableFuture.create(); + future.set(null); + when(mockXdsClient.getSubscribedResourcesMetadataSnapshot()).thenReturn(Futures.immediateFuture( + ImmutableMap.of())); - // For server connections, return a normally completed future - SettableFuture serverConnectionsFuture = SettableFuture.create(); - serverConnectionsFuture.set(null); - when(mockXdsClient.reportServerConnections(any())).thenReturn(serverConnectionsFuture); + // Create a future that will throw an exception + SettableFuture serverConnectionsFeature = SettableFuture.create(); + serverConnectionsFeature.setException(new Exception("test")); + when(mockXdsClient.reportServerConnections(any())).thenReturn(serverConnectionsFeature); reporter.setXdsClient(mockXdsClient); verify(mockMetricRecorder) .registerBatchCallback(gaugeBatchCallbackCaptor.capture(), any(), any()); gaugeBatchCallbackCaptor.getValue().accept(mockBatchRecorder); // Verify that the xdsClient methods were called - verify(mockXdsClient).reportResourceCounts(any()); + // verify(mockXdsClient).reportResourceCounts(any()); verify(mockXdsClient).reportServerConnections(any()); - assertThat(logs.size()).isEqualTo(1); - assertThat(logs.get(0).getLevel()).isEqualTo(Level.WARNING); - assertThat(logs.get(0).getMessage()).isEqualTo("Failed to report gauge metrics"); + assertThat(testLogHandler.getLogs().size()).isEqualTo(1); + assertThat(testLogHandler.getLogs().get(0).getLevel()).isEqualTo(Level.WARNING); + assertThat(testLogHandler.getLogs().get(0).getMessage()).isEqualTo( + "Failed to report gauge metrics"); logger.removeHandler(testLogHandler); } @@ -175,8 +174,8 @@ public void flush() {} public void metricGauges() { SettableFuture future = SettableFuture.create(); future.set(null); - when(mockXdsClient.reportResourceCounts(any(ResourceCallback.class))) - .thenReturn(future); + when(mockXdsClient.getSubscribedResourcesMetadataSnapshot()).thenReturn(Futures.immediateFuture( + ImmutableMap.of())); when(mockXdsClient.reportServerConnections(any(ServerConnectionCallback.class))) .thenReturn(future); reporter.setXdsClient(mockXdsClient); @@ -188,15 +187,14 @@ public void metricGauges() { // Trigger the internal call to reportCallbackMetrics() gaugeBatchCallback.accept(mockBatchRecorder); - ArgumentCaptor resourceCallbackCaptor = - ArgumentCaptor.forClass(ResourceCallback.class); ArgumentCaptor serverConnectionCallbackCaptor = ArgumentCaptor.forClass(ServerConnectionCallback.class); - verify(mockXdsClient).reportResourceCounts(resourceCallbackCaptor.capture()); + // verify(mockXdsClient).reportResourceCounts(resourceCallbackCaptor.capture()); verify(mockXdsClient).reportServerConnections(serverConnectionCallbackCaptor.capture()); // Get the captured callback - MetricReporterCallback callback = (MetricReporterCallback) resourceCallbackCaptor.getValue(); + MetricReporterCallback callback = (MetricReporterCallback) + serverConnectionCallbackCaptor.getValue(); // Verify that reportResourceCounts and reportServerConnections were called // with the captured callback @@ -230,6 +228,127 @@ public void metricReporterCallback() { eq(Collections.emptyList())); } + @Test + public void reportCallbackMetrics_computeAndReportResourceCounts() { + Map, Map> metadataByType = new HashMap<>(); + XdsResourceType listenerResource = XdsListenerResource.getInstance(); + XdsResourceType routeConfigResource = XdsRouteConfigureResource.getInstance(); + XdsResourceType clusterResource = XdsClusterResource.getInstance(); + + Any rawListener = + Any.pack(Listener.newBuilder().setName("listener.googleapis.com").build()); + long nanosLastUpdate = 1577923199_606042047L; + + Map ldsResourceMetadataMap = new HashMap<>(); + ldsResourceMetadataMap.put("resource1", + ResourceMetadata.newResourceMetadataRequested(false)); + ResourceMetadata ackedLdsResource = ResourceMetadata.newResourceMetadataAcked(rawListener, "42", + nanosLastUpdate); + ldsResourceMetadataMap.put("resource2", ackedLdsResource); + ldsResourceMetadataMap.put("resource3", + ResourceMetadata.newResourceMetadataAcked(rawListener, "43", nanosLastUpdate)); + ldsResourceMetadataMap.put("resource4", + ResourceMetadata.newResourceMetadataNacked(ackedLdsResource, "44", nanosLastUpdate, + "nacked after previous ack", true)); + + Map rdsResourceMetadataMap = new HashMap<>(); + ResourceMetadata requestedRdsResourceMetadata = ResourceMetadata.newResourceMetadataRequested( + false); + rdsResourceMetadataMap.put("resource5", + ResourceMetadata.newResourceMetadataNacked(requestedRdsResourceMetadata, "24", + nanosLastUpdate, "nacked after request", false)); + rdsResourceMetadataMap.put("resource6", + ResourceMetadata.newResourceMetadataDoesNotExist()); + + Map cdsResourceMetadataMap = new HashMap<>(); + cdsResourceMetadataMap.put("resource7", ResourceMetadata.newResourceMetadataUnknown()); + + metadataByType.put(listenerResource, ldsResourceMetadataMap); + metadataByType.put(routeConfigResource, rdsResourceMetadataMap); + metadataByType.put(clusterResource, cdsResourceMetadataMap); + + SettableFuture reportServerConnectionsCompleted = SettableFuture.create(); + reportServerConnectionsCompleted.set(null); + when(mockXdsClient.reportServerConnections(any(MetricReporterCallback.class))) + .thenReturn(reportServerConnectionsCompleted); + + ListenableFuture, Map>> + getResourceMetadataCompleted = Futures.immediateFuture(metadataByType); + when(mockXdsClient.getSubscribedResourcesMetadataSnapshot()) + .thenReturn(getResourceMetadataCompleted); + + reporter.reportCallbackMetrics(mockBatchRecorder, mockXdsClient); + + // LDS resource requested + verify(mockBatchRecorder).recordLongGauge(eqMetricInstrumentName("grpc.xds_client.resources"), + eq(1L), eq(Arrays.asList(target, "requested", listenerResource.typeUrl())), any()); + // LDS resources acked + verify(mockBatchRecorder).recordLongGauge(eqMetricInstrumentName("grpc.xds_client.resources"), + eq(2L), eq(Arrays.asList(target, "acked", listenerResource.typeUrl())), any()); + // LDS resource nacked but cached + verify(mockBatchRecorder).recordLongGauge(eqMetricInstrumentName("grpc.xds_client.resources"), + eq(1L), eq(Arrays.asList(target, "nacked_but_cached", listenerResource.typeUrl())), any()); + + // RDS resource nacked + verify(mockBatchRecorder).recordLongGauge(eqMetricInstrumentName("grpc.xds_client.resources"), + eq(1L), eq(Arrays.asList(target, "nacked", routeConfigResource.typeUrl())), any()); + // RDS resource does not exist + verify(mockBatchRecorder).recordLongGauge(eqMetricInstrumentName("grpc.xds_client.resources"), + eq(1L), eq(Arrays.asList(target, "does_not_exist", routeConfigResource.typeUrl())), any()); + + // CDS resource unknown + verify(mockBatchRecorder).recordLongGauge(eqMetricInstrumentName("grpc.xds_client.resources"), + eq(1L), eq(Arrays.asList(target, "unknown", clusterResource.typeUrl())), any()); + verifyNoMoreInteractions(mockBatchRecorder); + } + + @Test + public void reportCallbackMetrics_computeAndReportResourceCounts_emptyResources() { + Map, Map> metadataByType = new HashMap<>(); + XdsResourceType listenerResource = XdsListenerResource.getInstance(); + metadataByType.put(listenerResource, Collections.emptyMap()); + + SettableFuture reportServerConnectionsCompleted = SettableFuture.create(); + reportServerConnectionsCompleted.set(null); + when(mockXdsClient.reportServerConnections(any(MetricReporterCallback.class))) + .thenReturn(reportServerConnectionsCompleted); + + ListenableFuture, Map>> + getResourceMetadataCompleted = Futures.immediateFuture(metadataByType); + when(mockXdsClient.getSubscribedResourcesMetadataSnapshot()) + .thenReturn(getResourceMetadataCompleted); + + reporter.reportCallbackMetrics(mockBatchRecorder, mockXdsClient); + + // Verify that reportResourceCountGauge is never called + verifyNoInteractions(mockBatchRecorder); + } + + @Test + public void reportCallbackMetrics_computeAndReportResourceCounts_nullMetadata() { + TestlogHandler testLogHandler = new TestlogHandler(); + Logger logger = Logger.getLogger(XdsClientMetricReporterImpl.class.getName()); + logger.addHandler(testLogHandler); + + SettableFuture reportServerConnectionsCompleted = SettableFuture.create(); + reportServerConnectionsCompleted.set(null); + when(mockXdsClient.reportServerConnections(any(MetricReporterCallback.class))) + .thenReturn(reportServerConnectionsCompleted); + + ListenableFuture, Map>> + getResourceMetadataCompleted = Futures.immediateFailedFuture( + new Exception("Error generating metadata snapshot")); + when(mockXdsClient.getSubscribedResourcesMetadataSnapshot()) + .thenReturn(getResourceMetadataCompleted); + + reporter.reportCallbackMetrics(mockBatchRecorder, mockXdsClient); + assertThat(testLogHandler.getLogs().size()).isEqualTo(1); + assertThat(testLogHandler.getLogs().get(0).getLevel()).isEqualTo(Level.WARNING); + assertThat(testLogHandler.getLogs().get(0).getMessage()).isEqualTo( + "Failed to report gauge metrics"); + logger.removeHandler(testLogHandler); + } + @Test public void close_closesGaugeRegistration() { MetricSink.Registration mockRegistration = mock(MetricSink.Registration.class); @@ -253,4 +372,24 @@ public boolean matches(T instrument) { } }); } + + static class TestlogHandler extends Handler { + List logs = new ArrayList<>(); + + @Override + public void publish(LogRecord record) { + logs.add(record); + } + + @Override + public void close() {} + + @Override + public void flush() {} + + public List getLogs() { + return logs; + } + } + } From fb71c606405087d6b532e1657c36d3bb818cd72e Mon Sep 17 00:00:00 2001 From: Vindhya Ningegowda Date: Wed, 20 Nov 2024 09:01:27 -0800 Subject: [PATCH 09/13] fix typo --- .../test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java b/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java index a15221a7464..4fb77f0be42 100644 --- a/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java @@ -132,7 +132,7 @@ public void refCountedXdsClientObjectPool_getObjectCreatesNewInstanceIfAlreadySh provider.new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET, metricRecorder); XdsClient xdsClient1 = xdsClientPool.getObject(); assertThat(xdsClientPool.returnObject(xdsClient1)).isNull(); - assertThat(xdsClient1.isShutDogit wn()).isTrue(); + assertThat(xdsClient1.isShutDown()).isTrue(); XdsClient xdsClient2 = xdsClientPool.getObject(); assertThat(xdsClient2).isNotSameInstanceAs(xdsClient1); From 80a9c408868acb6efd34b3a6c6901f7195b3515f Mon Sep 17 00:00:00 2001 From: Vindhya Ningegowda Date: Wed, 20 Nov 2024 13:23:59 -0800 Subject: [PATCH 10/13] address review comment for javadoc --- xds/src/main/java/io/grpc/xds/client/XdsClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClient.java b/xds/src/main/java/io/grpc/xds/client/XdsClient.java index e3cef6a355f..f0ff676148b 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClient.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClient.java @@ -398,7 +398,7 @@ public interface ServerConnectionCallback { } /** - * Reports whether xDS client has a working ADS stream to xDS server. + * Reports whether xDS client has a "working" ADS stream to xDS server. * The definition of a working stream is defined in gRFC A78. * @see * A78-grpc-metrics-wrr-pf-xds.md From e04164b393919f8965efb5f893a1f83f76f261c1 Mon Sep 17 00:00:00 2001 From: Vindhya Ningegowda Date: Wed, 20 Nov 2024 14:19:35 -0800 Subject: [PATCH 11/13] Updated verifyResourceMetadataNacked method to verify ResourceMetadata.cached value --- .../java/io/grpc/xds/client/XdsClient.java | 4 +- .../io/grpc/xds/client/XdsClientImpl.java | 2 +- .../grpc/xds/GrpcXdsClientImplTestBase.java | 127 ++---------------- .../xds/XdsClientMetricReporterImplTest.java | 5 +- 4 files changed, 14 insertions(+), 124 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClient.java b/xds/src/main/java/io/grpc/xds/client/XdsClient.java index f0ff676148b..381d305789a 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClient.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClient.java @@ -174,8 +174,8 @@ public static ResourceMetadata newResourceMetadataUnknown() { return new ResourceMetadata(ResourceMetadataStatus.UNKNOWN, "", 0, false,null, null); } - public static ResourceMetadata newResourceMetadataRequested(boolean cached) { - return new ResourceMetadata(ResourceMetadataStatus.REQUESTED, "", 0, cached, null, null); + public static ResourceMetadata newResourceMetadataRequested() { + return new ResourceMetadata(ResourceMetadataStatus.REQUESTED, "", 0, false, null, null); } public static ResourceMetadata newResourceMetadataDoesNotExist() { diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java index 2c581f867d2..193216fffd0 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java @@ -635,7 +635,7 @@ public String toString() { } // Initial fetch scheduled or rescheduled, transition metadata state to REQUESTED. - metadata = ResourceMetadata.newResourceMetadataRequested(this.data != null); + metadata = ResourceMetadata.newResourceMetadataRequested(); respTimer = syncContext.schedule( new ResourceNotFound(), INITIAL_RESOURCE_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS, diff --git a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java index 555dfc10640..77815f5a991 100644 --- a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java +++ b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java @@ -107,9 +107,7 @@ import java.util.ArrayDeque; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Queue; import java.util.concurrent.BlockingDeque; @@ -487,10 +485,11 @@ private void verifyResourceMetadataAcked( private void verifyResourceMetadataNacked( XdsResourceType type, String resourceName, Any rawResource, String versionInfo, long updateTime, String failedVersion, long failedUpdateTimeNanos, - List failedDetails) { + List failedDetails, boolean cached) { ResourceMetadata resourceMetadata = verifyResourceMetadata(type, resourceName, rawResource, ResourceMetadataStatus.NACKED, versionInfo, updateTime, true); + assertThat(resourceMetadata.isCached()).isEqualTo(cached); UpdateFailureState errorState = resourceMetadata.getErrorState(); assertThat(errorState).isNotNull(); @@ -634,30 +633,6 @@ private void verifyServerFailureCount(int times, long serverFailureCount, String eq(xdsServer)); } - private void verifyResourceCountByCacheState(XdsResourceType type, String cacheState, - long resourceCount) { - Map metadataMap = awaitSubscribedResourcesMetadata().get(type); - assertThat(metadataMap).isNotNull(); - - // Calculate number of resources in cacheState - Map resourceCountsByState = new HashMap<>(); - for (ResourceMetadata metadata : metadataMap.values()) { - String cacheStateFromResourceStatus = cacheStateFromResourceStatus(metadata.getStatus(), - metadata.isCached()); - resourceCountsByState.compute(cacheStateFromResourceStatus, - (k, v) -> (v == null) ? 1 : v + 1); - } - - assertThat(resourceCountsByState.get(cacheState)).isEqualTo(resourceCount); - } - - private String cacheStateFromResourceStatus(ResourceMetadataStatus metadataStatus, - boolean isResourceCached) { - String status = metadataStatus.toString().toLowerCase(Locale.ROOT); - return metadataStatus == ResourceMetadataStatus.NACKED && isResourceCached - ? status + "_but_cached" : status; - } - /** * Invokes the callback, which will be called by {@link XdsClientMetricReporter} to record * whether XdsClient has a working ADS stream. @@ -803,7 +778,7 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid() { List errorsV2 = ImmutableList.of("LDS response Listener 'B' validation error: "); verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataNacked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, - VERSION_2, TIME_INCREMENT * 2, errorsV2); + VERSION_2, TIME_INCREMENT * 2, errorsV2, true); // Check metric data. verifyResourceValidInvalidCount(1, 1, 1, xdsServerInfo.target(), LDS.typeUrl()); if (!ignoreResourceDeletion()) { @@ -896,7 +871,7 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscripti verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 3); verifyResourceMetadataNacked( LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, VERSION_2, TIME_INCREMENT * 3, - errorsV2); + errorsV2, true); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(LDS, "C"); } else { @@ -1579,7 +1554,7 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { ImmutableList.of("RDS response RouteConfiguration 'B' validation error: "); verifyResourceMetadataAcked(RDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataNacked(RDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, - VERSION_2, TIME_INCREMENT * 2, errorsV2); + VERSION_2, TIME_INCREMENT * 2, errorsV2, true); verifyResourceMetadataAcked(RDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); call.verifyRequestNack(RDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); @@ -1963,9 +1938,9 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { List errorsV2 = ImmutableList.of("CDS response Cluster 'B' validation error: "); verifyResourceMetadataAcked(CDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataNacked(CDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, - VERSION_2, TIME_INCREMENT * 2, errorsV2); + VERSION_2, TIME_INCREMENT * 2, errorsV2, true); if (!ignoreResourceDeletion()) { - verifyResourceMetadataDoesNotExist(CDS, "C");; + verifyResourceMetadataDoesNotExist(CDS, "C"); } else { // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. verifyResourceMetadataAcked(CDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); @@ -2068,7 +2043,7 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti verifyResourceMetadataAcked(CDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 3); verifyResourceMetadataNacked( CDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, VERSION_2, TIME_INCREMENT * 3, - errorsV2); + errorsV2, true); if (!ignoreResourceDeletion()) { verifyResourceMetadataDoesNotExist(CDS, "C"); } else { @@ -2991,7 +2966,7 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { ImmutableList.of("EDS response ClusterLoadAssignment 'B' validation error: "); verifyResourceMetadataAcked(EDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataNacked(EDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, - VERSION_2, TIME_INCREMENT * 2, errorsV2); + VERSION_2, TIME_INCREMENT * 2, errorsV2, true); verifyResourceMetadataAcked(EDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); call.verifyRequestNack(EDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); @@ -4126,90 +4101,6 @@ public void serverFailureMetricReport_forRetryAndBackoff() { } - @Test - public void testResourceCounts() { - List subscribedResourceNames = ImmutableList.of("A", "B", "C"); - xdsClient.watchXdsResource(XdsListenerResource.getInstance(), "A", ldsResourceWatcher); - xdsClient.watchXdsResource(XdsListenerResource.getInstance(), "B", ldsResourceWatcher); - xdsClient.watchXdsResource(XdsListenerResource.getInstance(), "C", ldsResourceWatcher); - DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); - assertThat(call).isNotNull(); - verifyResourceMetadataRequested(LDS, "A"); - verifyResourceMetadataRequested(LDS, "B"); - verifyResourceMetadataRequested(LDS, "C"); - verifySubscribedResourcesMetadataSizes(3, 0, 0, 0); - // Check metric data. - verifyResourceCountByCacheState(LDS,"requested", 3); - - // LDS -> {A, B, C}, version 1 - ImmutableMap resourcesV1 = ImmutableMap.of( - "A", Any.pack(mf.buildListenerWithApiListenerForRds("A", "A.1")), - "B", Any.pack(mf.buildListenerWithApiListenerForRds("B", "B.1")), - "C", Any.pack(mf.buildListenerWithApiListenerForRds("C", "C.1"))); - call.sendResponse(LDS, resourcesV1.values().asList(), VERSION_1, "0000"); - // {A, B, C} -> ACK, version 1 - verifyResourceValidInvalidCount(1, 3, 0, xdsServerInfo.target(), LDS.typeUrl()); - verifyResourceMetadataAcked(LDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); - verifyResourceMetadataAcked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); - verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - // Check metric data. - verifyResourceCountByCacheState(LDS, "acked",3); - call.verifyRequest(LDS, subscribedResourceNames, VERSION_1, "0000", NODE); - - // LDS -> {A, B}, version 2 - // Failed to parse endpoint B - ImmutableMap resourcesV2 = ImmutableMap.of( - "A", Any.pack(mf.buildListenerWithApiListenerForRds("A", "A.2")), - "B", Any.pack(mf.buildListenerWithApiListenerInvalid("B"))); - call.sendResponse(LDS, resourcesV2.values().asList(), VERSION_2, "0001"); - // {A} -> ACK, version 2 - // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B - // {C} -> does not exist - verifyResourceValidInvalidCount(1, 1, 1, xdsServerInfo.target(), LDS.typeUrl()); - List errorsV2 = ImmutableList.of("LDS response Listener 'B' validation error: "); - verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); - verifyResourceMetadataNacked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, - VERSION_2, TIME_INCREMENT * 2, errorsV2); - if (!ignoreResourceDeletion()) { - verifyResourceMetadataDoesNotExist(LDS, "C"); - // Check metric data. - verifyResourceCountByCacheState(LDS, "acked", 1); - verifyResourceCountByCacheState(LDS, "nacked_but_cached", 1); - verifyResourceCountByCacheState(LDS, "does_not_exist", 1); - } else { - // When resource deletion is disabled, {C} stays ACKed in the previous version VERSION_1. - verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); - // Check metric data. - verifyResourceCountByCacheState(LDS, "acked", 2); - verifyResourceCountByCacheState(LDS, "nacked_but_cached", 1); - } - call.verifyRequestNack(LDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); - - // LDS -> {B, C} version 3 - ImmutableMap resourcesV3 = ImmutableMap.of( - "B", Any.pack(mf.buildListenerWithApiListenerForRds("B", "B.3")), - "C", Any.pack(mf.buildListenerWithApiListenerForRds("C", "C.3"))); - call.sendResponse(LDS, resourcesV3.values().asList(), VERSION_3, "0002"); - // {A} -> does not exist - // {B, C} -> ACK, version 3 - verifyResourceValidInvalidCount(1, 2, 0, xdsServerInfo.target(), LDS.typeUrl()); - if (!ignoreResourceDeletion()) { - verifyResourceMetadataDoesNotExist(LDS, "A"); - // Check metric data. - verifyResourceCountByCacheState(LDS, "acked", 2); - verifyResourceCountByCacheState(LDS, "does_not_exist", 1); - } else { - // When resource deletion is disabled, {A} stays ACKed in the previous version VERSION_2. - verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); - // Check metric data. - verifyResourceCountByCacheState(LDS, "acked", 3); - } - verifyResourceMetadataAcked(LDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); - verifyResourceMetadataAcked(LDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); - call.verifyRequest(LDS, subscribedResourceNames, VERSION_3, "0002", NODE); - verifySubscribedResourcesMetadataSizes(3, 0, 0, 0); - } - private XdsClientImpl createXdsClient(String serverUri) { BootstrapInfo bootstrapInfo = buildBootStrap(serverUri); return new XdsClientImpl( diff --git a/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java b/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java index 0723e958334..9ee3f88d921 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientMetricReporterImplTest.java @@ -241,7 +241,7 @@ public void reportCallbackMetrics_computeAndReportResourceCounts() { Map ldsResourceMetadataMap = new HashMap<>(); ldsResourceMetadataMap.put("resource1", - ResourceMetadata.newResourceMetadataRequested(false)); + ResourceMetadata.newResourceMetadataRequested()); ResourceMetadata ackedLdsResource = ResourceMetadata.newResourceMetadataAcked(rawListener, "42", nanosLastUpdate); ldsResourceMetadataMap.put("resource2", ackedLdsResource); @@ -252,8 +252,7 @@ public void reportCallbackMetrics_computeAndReportResourceCounts() { "nacked after previous ack", true)); Map rdsResourceMetadataMap = new HashMap<>(); - ResourceMetadata requestedRdsResourceMetadata = ResourceMetadata.newResourceMetadataRequested( - false); + ResourceMetadata requestedRdsResourceMetadata = ResourceMetadata.newResourceMetadataRequested(); rdsResourceMetadataMap.put("resource5", ResourceMetadata.newResourceMetadataNacked(requestedRdsResourceMetadata, "24", nanosLastUpdate, "nacked after request", false)); From 7dd44025c72e882079b7c9c9fef271989d771e68 Mon Sep 17 00:00:00 2001 From: Vindhya Ningegowda Date: Thu, 21 Nov 2024 09:31:07 -0800 Subject: [PATCH 12/13] Return Future instead of SettableFuture for reportServerConnections() and addressed review comments --- .../grpc/xds/XdsClientMetricReporterImpl.java | 40 +++++++++---------- .../java/io/grpc/xds/client/XdsClient.java | 20 ++++------ .../io/grpc/xds/client/XdsClientImpl.java | 3 +- 3 files changed, 28 insertions(+), 35 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java index e085bdba74c..1f095b82b79 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java @@ -18,8 +18,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.SettableFuture; -import io.grpc.Internal; import io.grpc.LongCounterMetricInstrument; import io.grpc.LongGaugeMetricInstrument; import io.grpc.MetricInstrumentRegistry; @@ -28,7 +26,6 @@ import io.grpc.MetricRecorder.BatchRecorder; import io.grpc.MetricRecorder.Registration; import io.grpc.xds.client.XdsClient; -import io.grpc.xds.client.XdsClient.ResourceCallback; import io.grpc.xds.client.XdsClient.ResourceMetadata; import io.grpc.xds.client.XdsClient.ResourceMetadata.ResourceMetadataStatus; import io.grpc.xds.client.XdsClient.ServerConnectionCallback; @@ -37,9 +34,9 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.Locale; import java.util.Map; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.logging.Level; @@ -49,7 +46,6 @@ /** * XdsClientMetricReporter implementation. */ -@Internal final class XdsClientMetricReporterImpl implements XdsClientMetricReporter { private static final Logger logger = Logger.getLogger( @@ -139,8 +135,7 @@ void close() { void reportCallbackMetrics(BatchRecorder recorder, XdsClient xdsClient) { MetricReporterCallback callback = new MetricReporterCallback(recorder, target); try { - SettableFuture reportServerConnectionsCompleted = - xdsClient.reportServerConnections(callback); + Future reportServerConnectionsCompleted = xdsClient.reportServerConnections(callback); ListenableFuture, Map>> getResourceMetadataCompleted = xdsClient.getSubscribedResourcesMetadataSnapshot(); @@ -148,12 +143,10 @@ void reportCallbackMetrics(BatchRecorder recorder, XdsClient xdsClient) { Map, Map> metadataByType = getResourceMetadataCompleted.get(10, TimeUnit.SECONDS); - SettableFuture reportResourceCountsCompleted = computeAndReportResourceCounts( - metadataByType, callback); + computeAndReportResourceCounts(metadataByType, callback); // Normally this shouldn't take long, but adding a timeout to avoid indefinite blocking - Void unused1 = reportServerConnectionsCompleted.get(5, TimeUnit.SECONDS); - Void unused2 = reportResourceCountsCompleted.get(5, TimeUnit.SECONDS); + Void unused = reportServerConnectionsCompleted.get(5, TimeUnit.SECONDS); } catch (ExecutionException | TimeoutException | InterruptedException e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); // re-set the current thread's interruption state @@ -162,11 +155,9 @@ void reportCallbackMetrics(BatchRecorder recorder, XdsClient xdsClient) { } } - private SettableFuture computeAndReportResourceCounts( + private void computeAndReportResourceCounts( Map, Map> metadataByType, MetricReporterCallback callback) { - SettableFuture future = SettableFuture.create(); - for (Map.Entry, Map> metadataByTypeEntry : metadataByType.entrySet()) { XdsResourceType type = metadataByTypeEntry.getKey(); @@ -180,20 +171,26 @@ private SettableFuture computeAndReportResourceCounts( resourceCountsByState.forEach((cacheState, count) -> callback.reportResourceCountGauge(count, cacheState, type.typeUrl())); } - future.set(null); - return future; } private static String cacheStateFromResourceStatus(ResourceMetadataStatus metadataStatus, boolean isResourceCached) { - String status = metadataStatus.toString().toLowerCase(Locale.ROOT); - return metadataStatus == ResourceMetadataStatus.NACKED && isResourceCached - ? status + "_but_cached" : status; + switch (metadataStatus) { + case REQUESTED: + return "requested"; + case DOES_NOT_EXIST: + return "does_not_exist"; + case ACKED: + return "acked"; + case NACKED: + return isResourceCached ? "nacked_but_cached" : "nacked"; + default: + return "unknown"; + } } @VisibleForTesting - static final class MetricReporterCallback implements ResourceCallback, - ServerConnectionCallback { + static final class MetricReporterCallback implements ServerConnectionCallback { private final BatchRecorder recorder; private final String target; @@ -203,7 +200,6 @@ static final class MetricReporterCallback implements ResourceCallback, } // TODO(dnvindhya): include the "authority" label once xds.authority is available. - @Override public void reportResourceCountGauge(long resourceCount, String cacheState, String resourceType) { recorder.recordLongGauge(RESOURCES_GAUGE, resourceCount, diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClient.java b/xds/src/main/java/io/grpc/xds/client/XdsClient.java index 381d305789a..06f15005c22 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClient.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClient.java @@ -24,7 +24,6 @@ import com.google.common.net.UrlEscapers; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; -import com.google.common.util.concurrent.SettableFuture; import com.google.protobuf.Any; import io.grpc.ExperimentalApi; import io.grpc.Status; @@ -37,6 +36,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.Executor; +import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.Nullable; @@ -386,24 +386,20 @@ public Map getServerLrsClientMap() { throw new UnsupportedOperationException(); } - /** Callback used to report gauge metric value for resources. */ - public interface ResourceCallback { - // TODO(dnvindhya): include the "authority" label once xds.authority is available. - void reportResourceCountGauge(long resourceCount, String cacheState, String resourceType); - } - /** Callback used to report a gauge metric value for server connections. */ public interface ServerConnectionCallback { void reportServerConnectionGauge(boolean isConnected, String xdsServer); } /** - * Reports whether xDS client has a "working" ADS stream to xDS server. - * The definition of a working stream is defined in gRFC A78. - * @see - * A78-grpc-metrics-wrr-pf-xds.md + * Reports whether xDS client has a "working" ADS stream to xDS server. The definition of a + * working stream is defined in gRFC A78. + * + * @see + * A78-grpc-metrics-wrr-pf-xds.md */ - public SettableFuture reportServerConnections(ServerConnectionCallback callback) { + public Future reportServerConnections(ServerConnectionCallback callback) { throw new UnsupportedOperationException(); } diff --git a/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java b/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java index 193216fffd0..529ac2747df 100644 --- a/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java +++ b/xds/src/main/java/io/grpc/xds/client/XdsClientImpl.java @@ -51,6 +51,7 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.Executor; +import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; @@ -529,7 +530,7 @@ private void handleResourceUpdate( } @Override - public SettableFuture reportServerConnections(ServerConnectionCallback callback) { + public Future reportServerConnections(ServerConnectionCallback callback) { SettableFuture future = SettableFuture.create(); syncContext.execute(() -> { serverCpClientMap.forEach((serverInfo, controlPlaneClient) -> From fc60ef91ee02aa3e104768da0d34fc1c593c7b58 Mon Sep 17 00:00:00 2001 From: Vindhya Ningegowda Date: Thu, 21 Nov 2024 13:40:27 -0800 Subject: [PATCH 13/13] Added unit test to verify ResourceMetadata.isCached() set to false on Nacked resource --- .../grpc/xds/XdsClientMetricReporterImpl.java | 2 +- .../grpc/xds/GrpcXdsClientImplTestBase.java | 30 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java index 1f095b82b79..fa88237a7ea 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientMetricReporterImpl.java @@ -200,7 +200,7 @@ static final class MetricReporterCallback implements ServerConnectionCallback { } // TODO(dnvindhya): include the "authority" label once xds.authority is available. - public void reportResourceCountGauge(long resourceCount, String cacheState, + void reportResourceCountGauge(long resourceCount, String cacheState, String resourceType) { recorder.recordLongGauge(RESOURCES_GAUGE, resourceCount, Arrays.asList(target, cacheState, resourceType), Collections.emptyList()); diff --git a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java index 77815f5a991..198faea7fdc 100644 --- a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java +++ b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java @@ -1666,6 +1666,36 @@ public void rdsResourceUpdated() { TIME_INCREMENT * 2); } + @Test + public void rdsResourceInvalid() { + xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), "A", rdsResourceWatcher); + xdsClient.watchXdsResource(XdsRouteConfigureResource.getInstance(), "B", rdsResourceWatcher); + DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); + assertThat(call).isNotNull(); + verifyResourceMetadataRequested(RDS, "A"); + verifyResourceMetadataRequested(RDS, "B"); + verifySubscribedResourcesMetadataSizes(0, 0, 2, 0); + + // RDS -> {A, B}, version 1 + // Failed to parse endpoint B + List vhostsV1 = mf.buildOpaqueVirtualHosts(1); + ImmutableMap resourcesV1 = ImmutableMap.of( + "A", Any.pack(mf.buildRouteConfiguration("A", vhostsV1)), + "B", Any.pack(mf.buildRouteConfigurationInvalid("B"))); + call.sendResponse(RDS, resourcesV1.values().asList(), VERSION_1, "0000"); + + // {A} -> ACK, version 1 + // {B} -> NACK, version 1, rejected version 1, rejected reason: Failed to parse B + List errorsV1 = + ImmutableList.of("RDS response RouteConfiguration 'B' validation error: "); + verifyResourceMetadataAcked(RDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); + verifyResourceMetadataNacked(RDS, "B", null, "", 0, + VERSION_1, TIME_INCREMENT, errorsV1, false); + // Check metric data. + verifyResourceValidInvalidCount(1, 1, 1, xdsServerInfo.target(), RDS.typeUrl()); + verifySubscribedResourcesMetadataSizes(0, 0, 2, 0); + } + @Test public void rdsResourceDeletedByLdsApiListener() { xdsClient.watchXdsResource(XdsListenerResource.getInstance(), LDS_RESOURCE,