From 63f243f005861fd1949937ffd211afdf13ffe4c2 Mon Sep 17 00:00:00 2001 From: Kai Hudalla Date: Sun, 3 Dec 2023 16:41:03 +0100 Subject: [PATCH] [#3517] Support filtering for gateway or edge devices only The MongoDB based registry has been extended to support the isGateway query parameter of the Registry Management API's search devices operation. --- .../AbstractDeviceManagementService.java | 7 +- .../hono/service/management/Filter.java | 53 +++++- .../device/DeviceManagementService.java | 8 +- .../AbstractDeviceManagementServiceTest.java | 30 ++- ...ractDeviceManagementSearchDevicesTest.java | 49 +++++ .../mongodb/model/DeviceDao.java | 3 + .../mongodb/model/MongoDbBasedDeviceDao.java | 119 ++++++++++-- .../MongoDbBasedDeviceManagementService.java | 3 +- .../mongodb/utils/MongoDbDocumentBuilder.java | 85 ++++++++- .../content/user-guide/device-registry.md | 16 +- site/homepage/content/release-notes.md | 10 +- .../hono/tests/DeviceRegistryHttpClient.java | 4 + .../tests/registry/DeviceManagementIT.java | 172 +++++++++++++++--- 13 files changed, 475 insertions(+), 84 deletions(-) diff --git a/services/device-registry-base/src/main/java/org/eclipse/hono/deviceregistry/service/device/AbstractDeviceManagementService.java b/services/device-registry-base/src/main/java/org/eclipse/hono/deviceregistry/service/device/AbstractDeviceManagementService.java index e82b05bc7f..2eb134b444 100644 --- a/services/device-registry-base/src/main/java/org/eclipse/hono/deviceregistry/service/device/AbstractDeviceManagementService.java +++ b/services/device-registry-base/src/main/java/org/eclipse/hono/deviceregistry/service/device/AbstractDeviceManagementService.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2020, 2023 Contributors to the Eclipse Foundation + * Copyright (c) 2020 Contributors to the Eclipse Foundation * * See the NOTICE file(s) distributed with this work for additional * information regarding copyright ownership. @@ -225,8 +225,8 @@ protected Future> processDeleteDevicesOfTenant(final String tenantI * retrieve the whole result set page by page. * @param filters A list of filters. The filters are predicates that objects in the result set must match. * @param sortOptions A list of sort options. The sortOptions specify properties to sort the result set by. - * @param isGateway Optional filter for searching only gateways or only devices. - * If given parameter is Optional.empty() result will contain both gateways and devices. + * @param isGateway A filter for restricting the search to gateway ({@code True}) or edge ({@code False} devices only. + * If empty, the search will not be restricted. * @param span The active OpenTracing span to use for tracking this operation. *

* Implementations must not invoke the {@link Span#finish()} nor the {@link Span#finish(long)} @@ -417,6 +417,7 @@ public final Future>> searchDevices( Objects.requireNonNull(tenantId); Objects.requireNonNull(filters); Objects.requireNonNull(sortOptions); + Objects.requireNonNull(isGateway); Objects.requireNonNull(span); if (pageSize <= 0) { diff --git a/services/device-registry-base/src/main/java/org/eclipse/hono/service/management/Filter.java b/services/device-registry-base/src/main/java/org/eclipse/hono/service/management/Filter.java index a85f497b25..af0640905b 100644 --- a/services/device-registry-base/src/main/java/org/eclipse/hono/service/management/Filter.java +++ b/services/device-registry-base/src/main/java/org/eclipse/hono/service/management/Filter.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2020, 2022 Contributors to the Eclipse Foundation + * Copyright (c) 2020 Contributors to the Eclipse Foundation * * See the NOTICE file(s) distributed with this work for additional * information regarding copyright ownership. @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.quarkus.runtime.annotations.RegisterForReflection; +import io.vertx.core.json.JsonArray; import io.vertx.core.json.pointer.JsonPointer; /** @@ -37,27 +38,59 @@ public final class Filter { private Operator operator = Operator.eq; /** - * An enum defining supported filter operators. + * Supported filter operators. */ public enum Operator { - eq + eq, + in, + not_in + } + + private Filter(final String field, final Object value, final Operator op) { + Objects.requireNonNull(field); + Objects.requireNonNull(value); + this.field = JsonPointer.from(field); + this.value = value; + this.operator = Optional.ofNullable(op).orElse(Operator.eq); } /** - * Creates a filter for a field and value using the equals operator. + * Creates a filter for a field and value using the equals operator. * - * @param field The field to use for filtering. + * @param field A JSON Pointer to the field to use for filtering. * @param value The value corresponding to the field to use for filtering. - * @throws IllegalArgumentException if the field is not a valid pointer. + * @throws IllegalArgumentException if the field is not a valid JSON pointer. * @throws NullPointerException if any of the parameters is {@code null}. */ public Filter(@JsonProperty(value = RegistryManagementConstants.FIELD_FILTER_FIELD, required = true) final String field, @JsonProperty(value = RegistryManagementConstants.FIELD_FILTER_VALUE, required = true) final Object value) { - Objects.requireNonNull(field); - Objects.requireNonNull(value); + this(field, value, Operator.eq); + } - this.field = JsonPointer.from(field); - this.value = value; + /** + * Creates a filter for a field and value list using the in operator. + * + * @param field A JSON Pointer to the field to use for filtering. + * @param valueList The list of values to match. + * @return The filter. + * @throws IllegalArgumentException if the field is not a valid pointer. + * @throws NullPointerException if any of the parameters are {@code null}. + */ + public static Filter inFilter(final String field, final JsonArray valueList) { + return new Filter(field, valueList, Operator.in); + } + + /** + * Creates a filter for a field and value list using the not in operator. + * + * @param field A JSON Pointer to the field to use for filtering. + * @param valueList The list of values to match. + * @return The filter. + * @throws IllegalArgumentException if the field is not a valid pointer. + * @throws NullPointerException if any of the parameters are {@code null}. + */ + public static Filter notInFilter(final String field, final JsonArray valueList) { + return new Filter(field, valueList, Operator.not_in); } /** diff --git a/services/device-registry-base/src/main/java/org/eclipse/hono/service/management/device/DeviceManagementService.java b/services/device-registry-base/src/main/java/org/eclipse/hono/service/management/device/DeviceManagementService.java index 02a5058550..99731d964e 100644 --- a/services/device-registry-base/src/main/java/org/eclipse/hono/service/management/device/DeviceManagementService.java +++ b/services/device-registry-base/src/main/java/org/eclipse/hono/service/management/device/DeviceManagementService.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2016, 2023 Contributors to the Eclipse Foundation + * Copyright (c) 2016 Contributors to the Eclipse Foundation * * See the NOTICE file(s) distributed with this work for additional * information regarding copyright ownership. @@ -102,14 +102,14 @@ public interface DeviceManagementService { * Implementations must not invoke the {@link Span#finish()} nor the {@link Span#finish(long)} * methods. However,implementations may log (error) events on this span, set tags and use this span * as the parent for additional spans created as part of this method's execution. - * @param isGateway Optional filter for searching only gateways or only devices. - * If given parameter is Optional.empty() result will contain both gateways and devices. + * @param isGateway A filter for restricting the search to gateway ({@code True}) or edge ({@code False} devices only. + * If empty, the search will not be restricted. * @return A future indicating the outcome of the operation. *

* The future will be succeeded with a result containing the matching devices. Otherwise, the future will * be failed with a {@link org.eclipse.hono.client.ServiceInvocationException} containing an error code * as specified in the Device Registry Management API. - * @throws NullPointerException if any of filters, sort options or tracing span are {@code null}. + * @throws NullPointerException if any of filters, sort options, gateway filter or tracing span are {@code null}. * @throws IllegalArgumentException if page size is <= 0 or page offset is < 0. * @see Device Registry * Management API - Search Devices diff --git a/services/device-registry-base/src/test/java/org/eclipse/hono/deviceregistry/service/device/AbstractDeviceManagementServiceTest.java b/services/device-registry-base/src/test/java/org/eclipse/hono/deviceregistry/service/device/AbstractDeviceManagementServiceTest.java index 843ae07bc0..da27a38349 100644 --- a/services/device-registry-base/src/test/java/org/eclipse/hono/deviceregistry/service/device/AbstractDeviceManagementServiceTest.java +++ b/services/device-registry-base/src/test/java/org/eclipse/hono/deviceregistry/service/device/AbstractDeviceManagementServiceTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2022 Contributors to the Eclipse Foundation + * Copyright (c) 2021 Contributors to the Eclipse Foundation * * See the NOTICE file(s) distributed with this work for additional * information regarding copyright ownership. @@ -13,6 +13,8 @@ package org.eclipse.hono.deviceregistry.service.device; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -23,6 +25,7 @@ import static com.google.common.truth.Truth.assertThat; import java.net.HttpURLConnection; +import java.util.List; import java.util.Optional; import org.eclipse.hono.notification.NotificationEventBusSupport; @@ -67,6 +70,31 @@ void setUp() { deviceManagementService = new TestDeviceManagementService(vertx); } + /** + * Verifies that the search devices operation verifies that all required parameters + * are non {@code null}. + */ + @Test + public void testSearchDevicesRejectsNullParameters() { + assertAll( + () -> assertThrows( + NullPointerException.class, + () -> deviceManagementService.searchDevices( + DEFAULT_TENANT_ID, 10, 0, null, List.of(), Optional.empty(), SPAN) + ), + () -> assertThrows( + NullPointerException.class, + () -> deviceManagementService.searchDevices( + DEFAULT_TENANT_ID, 10, 0, List.of(), null, Optional.empty(), SPAN) + ), + () -> assertThrows( + NullPointerException.class, + () -> deviceManagementService.searchDevices( + DEFAULT_TENANT_ID, 10, 0, List.of(), List.of(), null, SPAN) + ) + ); + } + /** * Verifies that {@link AbstractDeviceManagementService#createDevice(String, Optional, Device, Span)} publishes the * expected notification. diff --git a/services/device-registry-base/src/test/java/org/eclipse/hono/service/management/device/AbstractDeviceManagementSearchDevicesTest.java b/services/device-registry-base/src/test/java/org/eclipse/hono/service/management/device/AbstractDeviceManagementSearchDevicesTest.java index 4f6e384135..21053b47ad 100644 --- a/services/device-registry-base/src/test/java/org/eclipse/hono/service/management/device/AbstractDeviceManagementSearchDevicesTest.java +++ b/services/device-registry-base/src/test/java/org/eclipse/hono/service/management/device/AbstractDeviceManagementSearchDevicesTest.java @@ -25,6 +25,8 @@ import org.eclipse.hono.service.management.SearchResult; import org.eclipse.hono.service.management.Sort; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import io.opentracing.noop.NoopSpan; import io.vertx.core.Future; @@ -108,6 +110,53 @@ default void testSearchDevicesWithAFilterSucceeds(final VertxTestContext ctx) { })); } + /** + * Verifies that a request to search gateway devices succeeds and matching devices are found. + * + * @param isGateway {@code true} if only gateways should be searched for. + * @param expectedFirstDeviceId The identifier that the first device in the result set is expected to have. + * @param expectedSecondDeviceId The identifier that the second device in the result set is expected to have. + * @param ctx The vert.x test context. + */ + @ParameterizedTest + @CsvSource(value = { "true, testDevice2, testDevice3", "false, testDevice0, testDevice1" }) + default void testSearchGatewayDevicesWithFilterSucceeds( + final boolean isGateway, + final String expectedFirstDeviceId, + final String expectedSecondDeviceId, + final VertxTestContext ctx) { + final String tenantId = DeviceRegistryUtils.getUniqueIdentifier(); + final int pageSize = 10; + final int pageOffset = 0; + final var filter = List.of(new Filter("/enabled", false)); + final var sortOptions = List.of(new Sort("/id")); + + createDevices(tenantId, Map.of( + "testDevice0", new Device().setEnabled(false).setMemberOf(List.of()), + "testDevice1", new Device().setEnabled(false).setVia(List.of("testDevice2")), + "testDevice2", new Device().setEnabled(false), + "testDevice3", new Device().setEnabled(false).setMemberOf(List.of("gwGroup")))) + .compose(ok -> getDeviceManagementService().searchDevices( + tenantId, + pageSize, + pageOffset, + filter, + sortOptions, + Optional.of(isGateway), + NoopSpan.INSTANCE)) + .onComplete(ctx.succeeding(s -> { + ctx.verify(() -> { + assertThat(s.getStatus()).isEqualTo(HttpURLConnection.HTTP_OK); + + final SearchResult searchResult = s.getPayload(); + assertThat(searchResult.getTotal()).isEqualTo(2); + assertThat(searchResult.getResult().get(0).getId()).isEqualTo(expectedFirstDeviceId); + assertThat(searchResult.getResult().get(1).getId()).isEqualTo(expectedSecondDeviceId); + }); + ctx.completeNow(); + })); + } + /** * Verifies that a request to search devices with multiple filters succeeds and matching devices are found. * diff --git a/services/device-registry-mongodb/src/main/java/org/eclipse/hono/deviceregistry/mongodb/model/DeviceDao.java b/services/device-registry-mongodb/src/main/java/org/eclipse/hono/deviceregistry/mongodb/model/DeviceDao.java index 54b648ef8c..3089988457 100644 --- a/services/device-registry-mongodb/src/main/java/org/eclipse/hono/deviceregistry/mongodb/model/DeviceDao.java +++ b/services/device-registry-mongodb/src/main/java/org/eclipse/hono/deviceregistry/mongodb/model/DeviceDao.java @@ -89,6 +89,8 @@ public interface DeviceDao { * retrieve the whole result set page by page. * @param filters A list of filters. The filters are predicates that objects in the result set must match. * @param sortOptions A list of sort options. The sortOptions specify properties to sort the result set by. + * @param isGateway A filter for restricting the search to gateway ({@code True}) or edge ({@code False} devices only. + * If empty, the search will not be restricted. * @param tracingContext The context to track the processing of the request in * or {@code null} if no such context exists. * @return A future indicating the outcome of the operation. @@ -105,6 +107,7 @@ Future> find( int pageOffset, List filters, List sortOptions, + Optional isGateway, SpanContext tracingContext); /** diff --git a/services/device-registry-mongodb/src/main/java/org/eclipse/hono/deviceregistry/mongodb/model/MongoDbBasedDeviceDao.java b/services/device-registry-mongodb/src/main/java/org/eclipse/hono/deviceregistry/mongodb/model/MongoDbBasedDeviceDao.java index 668beb6767..7bba74fac9 100644 --- a/services/device-registry-mongodb/src/main/java/org/eclipse/hono/deviceregistry/mongodb/model/MongoDbBasedDeviceDao.java +++ b/services/device-registry-mongodb/src/main/java/org/eclipse/hono/deviceregistry/mongodb/model/MongoDbBasedDeviceDao.java @@ -15,6 +15,8 @@ package org.eclipse.hono.deviceregistry.mongodb.model; import java.net.HttpURLConnection; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -103,11 +105,26 @@ public Future createIndices() { return createIndex( new JsonObject().put(BaseDto.FIELD_TENANT_ID, 1).put(DeviceDto.FIELD_DEVICE_ID, 1), new IndexOptions().unique(true)) - .onSuccess(ok -> indicesCreated.set(true)) - .onComplete(r -> { - creatingIndices.set(false); - result.handle(r); - }); + .compose(ok -> createIndex( + new JsonObject().put(BaseDto.FIELD_TENANT_ID, 1), + new IndexOptions() + .name("tenant_id.via_exists") + .partialFilterExpression(MongoDbDocumentBuilder.builder() + .withGatewayDevices() + .document()))) + .compose(ok -> createIndex( + new JsonObject().put(BaseDto.FIELD_TENANT_ID, 1), + new IndexOptions() + .name("tenant_id.memberof_exists") + .partialFilterExpression( + new JsonObject().put( + MongoDbDocumentBuilder.DEVICE_MEMBEROF_PATH, + new JsonObject().put("$exists", true))))) + .onSuccess(ok -> indicesCreated.set(true)) + .onComplete(r -> { + creatingIndices.set(false); + result.handle(r); + }); } else { LOG.debug("already trying to create indices"); } @@ -284,11 +301,13 @@ public Future> find( final int pageOffset, final List filters, final List sortOptions, + final Optional isGateway, final SpanContext tracingContext) { Objects.requireNonNull(tenantId); Objects.requireNonNull(filters); Objects.requireNonNull(sortOptions); + Objects.requireNonNull(isGateway); if (pageSize <= 0) { throw new IllegalArgumentException("page size must be a positive integer"); @@ -301,22 +320,40 @@ public Future> find( .addReference(References.CHILD_OF, tracingContext) .start(); - final JsonObject filterDocument = MongoDbDocumentBuilder.builder() - .withTenantId(tenantId) - .withDeviceFilters(filters) - .document(); + final Promise> filtersPromise = Promise.promise(); + if (isGateway.isPresent()) { + getIdsOfGatewayDevices(tenantId, span) + .map(gatewayIds -> { + final List effectiveFilters = new ArrayList<>(filters); + if (isGateway.get()) { + effectiveFilters.add(Filter.inFilter("/id", gatewayIds)); + } else { + effectiveFilters.add(Filter.notInFilter("/id", gatewayIds)); + } + return effectiveFilters; + }) + .andThen(filtersPromise); + } else { + filtersPromise.complete(filters); + } + final JsonObject sortDocument = MongoDbDocumentBuilder.builder() .withDeviceSortOptions(sortOptions) .document(); - return processSearchResource( - pageSize, - pageOffset, - filterDocument, - sortDocument, - MongoDbBasedDeviceDao::getDevicesWithId) - .onFailure(t -> TracingHelper.logError(span, "error finding devices", t)) - .onComplete(r -> span.finish()); + return filtersPromise.future() + .map(effectiveFilters -> MongoDbDocumentBuilder.builder() + .withTenantId(tenantId) + .withDeviceFilters(effectiveFilters) + .document()) + .compose(filterDocument -> processSearchResource( + pageSize, + pageOffset, + filterDocument, + sortDocument, + MongoDbBasedDeviceDao::getDevicesWithId)) + .onFailure(t -> TracingHelper.logError(span, "error finding devices", t)) + .onComplete(r -> span.finish()); } private static List getDevicesWithId(final JsonObject searchResult) { @@ -484,4 +521,52 @@ public Future count(final String tenantId, final SpanContext tracingContex .recover(this::mapError) .onComplete(r -> span.finish()); } + + private Future getIdsOfGatewayDevices(final String tenantId, final Span span) { + + final var viaQuery = MongoDbDocumentBuilder.builder() + .withTenantId(tenantId) + .withGatewayDevices() + .document(); + + final var viaElements = mongoClient.distinctWithQuery( + collectionName, + MongoDbDocumentBuilder.DEVICE_VIA_PATH, + String.class.getName(), + viaQuery) + .map(array -> array.stream() + .filter(String.class::isInstance) + .map(String.class::cast) + .collect(Collectors.toSet())); + + final var gatewayGroupQuery = MongoDbDocumentBuilder.builder() + .withTenantId(tenantId) + .withMemberOfAnyGatewayGroup() + .document(); + + final var gwGroupMembers = mongoClient.distinctWithQuery( + collectionName, + DeviceDto.FIELD_DEVICE_ID, + String.class.getName(), + gatewayGroupQuery) + .map(array -> array.stream() + .filter(String.class::isInstance) + .map(String.class::cast) + .collect(Collectors.toSet())); + + return Future.all(viaElements, gwGroupMembers) + .map(ok -> { + final Set result = new HashSet<>(viaElements.result()); + result.addAll(gwGroupMembers.result()); + return new JsonArray(new ArrayList<>(result)); + }) + .onFailure(t -> LOG.debug("error retrieving gateway device IDs for tenant {}", tenantId, t)) + .onSuccess(idList -> { + if (LOG.isDebugEnabled()) { + LOG.debug("found gateway device IDs for tenant {}: {}", + tenantId, idList.encodePrettily()); + } + }) + .recover(this::mapError); + } } diff --git a/services/device-registry-mongodb/src/main/java/org/eclipse/hono/deviceregistry/mongodb/service/MongoDbBasedDeviceManagementService.java b/services/device-registry-mongodb/src/main/java/org/eclipse/hono/deviceregistry/mongodb/service/MongoDbBasedDeviceManagementService.java index c2f074d47d..abd979fc3c 100644 --- a/services/device-registry-mongodb/src/main/java/org/eclipse/hono/deviceregistry/mongodb/service/MongoDbBasedDeviceManagementService.java +++ b/services/device-registry-mongodb/src/main/java/org/eclipse/hono/deviceregistry/mongodb/service/MongoDbBasedDeviceManagementService.java @@ -145,10 +145,11 @@ protected Future>> processSearchDevic Objects.requireNonNull(tenantId); Objects.requireNonNull(filters); Objects.requireNonNull(sortOptions); + Objects.requireNonNull(isGateway); Objects.requireNonNull(span); return tenantInformationService.getTenant(tenantId, span) - .compose(ok -> deviceDao.find(tenantId, pageSize, pageOffset, filters, sortOptions, span.context())) + .compose(ok -> deviceDao.find(tenantId, pageSize, pageOffset, filters, sortOptions, isGateway, span.context())) .map(result -> OperationResult.ok( HttpURLConnection.HTTP_OK, result, diff --git a/services/device-registry-mongodb/src/main/java/org/eclipse/hono/deviceregistry/mongodb/utils/MongoDbDocumentBuilder.java b/services/device-registry-mongodb/src/main/java/org/eclipse/hono/deviceregistry/mongodb/utils/MongoDbDocumentBuilder.java index ea16092a20..70ebb15f50 100644 --- a/services/device-registry-mongodb/src/main/java/org/eclipse/hono/deviceregistry/mongodb/utils/MongoDbDocumentBuilder.java +++ b/services/device-registry-mongodb/src/main/java/org/eclipse/hono/deviceregistry/mongodb/utils/MongoDbDocumentBuilder.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2020, 2022 Contributors to the Eclipse Foundation + * Copyright (c) 2020 Contributors to the Eclipse Foundation * * See the NOTICE file(s) distributed with this work for additional * information regarding copyright ownership. @@ -37,6 +37,22 @@ */ public final class MongoDbDocumentBuilder { + /** + * The path to a Device's {@value RegistryManagementConstants#FIELD_VIA} property. + */ + public static final String DEVICE_VIA_PATH = String.format( + "%s.%s", + DeviceDto.FIELD_DEVICE, + RegistryManagementConstants.FIELD_VIA); + /** + * The path to a Device's {@value RegistryManagementConstants#FIELD_MEMBER_OF} property. + */ + public static final String DEVICE_MEMBEROF_PATH = String.format( + "%s.%s", + DeviceDto.FIELD_DEVICE, + RegistryManagementConstants.FIELD_MEMBER_OF); + + private static final JsonArray EMPTY_ARRAY = new JsonArray(); private static final JsonPointer FIELD_ID = JsonPointer.from("/id"); private static final String TENANT_ALIAS_PATH = String.format( "%s.%s", @@ -58,8 +74,10 @@ public final class MongoDbDocumentBuilder { private static final String MONGODB_OPERATOR_ELEM_MATCH = "$elemMatch"; private static final String MONGODB_OPERATOR_EXISTS = "$exists"; private static final String MONGODB_OPERATOR_IN = "$in"; + private static final String MONGODB_OPERATOR_NOT = "$not"; private static final String MONGODB_OPERATOR_NOT_EQUALS = "$ne"; private static final String MONGODB_OPERATOR_OR = "$or"; + private static final String MONGODB_OPERATOR_REGEX = "$regex"; private final JsonObject document; @@ -96,6 +114,33 @@ public MongoDbDocumentBuilder withTenantId(final String tenantId) { return this; } + /** + * Adds filter criteria that matches documents containing a list of gateway devices. + * + * @return a reference to this for fluent use. + */ + public MongoDbDocumentBuilder withGatewayDevices() { + document.put( + DEVICE_VIA_PATH, + new JsonObject().put(MONGODB_OPERATOR_EXISTS, true)); + return this; + } + + /** + * Adds filter criteria that matches documents containing a non-empty list of gateway groups + * that the device is a member of. + * + * @return a reference to this for fluent use. + */ + public MongoDbDocumentBuilder withMemberOfAnyGatewayGroup() { + document.put( + DEVICE_MEMBEROF_PATH, + new JsonObject() + .put(MONGODB_OPERATOR_EXISTS, true) + .put(MONGODB_OPERATOR_NOT_EQUALS, EMPTY_ARRAY)); + return this; + } + /** * Adds filter criteria that matches documents containing a given tenant ID. *

@@ -275,14 +320,40 @@ public MongoDbDocumentBuilder withTenantSortOptions(final List sortOptions return this; } + private void applyEqFilter(final Filter filter, final Function fieldMapper) { + if (filter.getValue() instanceof String stringValue) { + document.put(fieldMapper.apply(filter.getField()), new JsonObject().put(MONGODB_OPERATOR_REGEX, + DeviceRegistryUtils.getRegexExpressionForSearchOperation(stringValue))); + } else { + document.put(fieldMapper.apply(filter.getField()), filter.getValue()); + } + } + + private void applyInFilter(final Filter filter, final Function fieldMapper) { + if (filter.getValue() instanceof JsonArray valueList) { + final var expr = new JsonObject().put(MONGODB_OPERATOR_IN, valueList); + switch (filter.getOperator()) { + case in: + document.put(fieldMapper.apply(filter.getField()), expr); + break; + case not_in: + document.put(fieldMapper.apply(filter.getField()), new JsonObject().put(MONGODB_OPERATOR_NOT, expr)); + break; + default: + // we can only handle $in operator + } + } + } + private void applySearchFilters(final List filters, final Function fieldMapper) { filters.forEach(filter -> { - if (filter.getValue() instanceof String) { - final String value = (String) filter.getValue(); - document.put(fieldMapper.apply(filter.getField()), new JsonObject().put("$regex", - DeviceRegistryUtils.getRegexExpressionForSearchOperation(value))); - } else { - document.put(fieldMapper.apply(filter.getField()), filter.getValue()); + switch (filter.getOperator()) { + case eq: + applyEqFilter(filter, fieldMapper); + break; + case in, not_in: + applyInFilter(filter, fieldMapper); + break; } }); } diff --git a/site/documentation/content/user-guide/device-registry.md b/site/documentation/content/user-guide/device-registry.md index cba11f8aba..51706a878e 100644 --- a/site/documentation/content/user-guide/device-registry.md +++ b/site/documentation/content/user-guide/device-registry.md @@ -57,11 +57,11 @@ The tenants in the registry can be managed using the Device Registry Management [tenant related resources]({{< relref "/api/management#tenants" >}}). {{% notice info %}} -The JDBC based registry implementation does not support the following features: +The JDBC based registry implementation currently has the following limitations: * Tenants can be retrieved using the [search tenants]({{< relref "/api/management#tenants/searchTenants" >}}) operation defined by the Device Registry Management API, but the *filterJson* and *sortJson* query parameters are - (currently) being ignored. The result set will always be sorted by the tenant Id in ascending order. + (currently) being ignored. The result set will always be sorted by the tenant ID in ascending order. * The *alias* and *trust-anchor-group* properties defined on a tenant are being ignored by the registry. Consequently, multiple tenants can not be configured to use the same trust anchor(s). {{% /notice %}} @@ -138,19 +138,13 @@ The devices in the registry can be managed using the Device Registry Management [device related resources]({{< relref "/api/management#devices" >}}). {{% notice info %}} -The JDBC based registry implementation does not support the following features: +The JDBC based registry implementation currently has the following limitations: * Registration information can be retrieved using the [search devices]({{< relref "/api/management#devices/searchDevicesForTenant" >}}) operation defined by the Device - Registry Management API. The *filterJson* query parameter currently only allows one filter expression per request, + Registry Management API. The *filterJson* query parameter currently only allows one filter expression per request, the *sortJson* query parameter is (currently) being ignored. - The result set will always be sorted by the device Id in ascending order. - -The Mongo DB based registry implementation does not support the following features: - -* Registration information can be retrieved using the - [search devices]({{< relref "/api/management#devices/searchDevicesForTenant" >}}) operation defined by the Device - Registry Management API. The *isGateway* query parameter is (currently) being ignored. + The result set will always be sorted by the device ID in ascending order. {{% /notice %}} ### Managing Credentials diff --git a/site/homepage/content/release-notes.md b/site/homepage/content/release-notes.md index 6ce77daffe..995a7d1047 100644 --- a/site/homepage/content/release-notes.md +++ b/site/homepage/content/release-notes.md @@ -8,9 +8,9 @@ description = "Information about changes in recent Hono releases. Includes new f ### Fixes & Enhancements -* When running in a Kubernetes cluster with nodes using cgroups v2, the 'hono.command_internal.*' Kafka topics were not +* When running in a Kubernetes cluster with nodes using cgroups v2, the `hono.command_internal.*` Kafka topics were not being cleaned up. This has been fixed. Note that the solution requires the Hono protocol adapter pods to have - a service account with an assigned RBAC role that allows to perform "get" on the "pods" resource. + a service account with an assigned RBAC role that allows to perform `get` on the `pods` resource. * When using Pub/Sub messaging, there were potentially issues concerning the AMQP connection between protocol adapter and command router, leading for example to timeouts when MQTT devices subscribed/unsubscribed to the command topic. This has been fixed. @@ -18,7 +18,11 @@ description = "Information about changes in recent Hono releases. Includes new f instance and thus simplifies test setup and configuration. * The command line client was still trying to connect to the insecure ports of the Sandbox. This has been changed so that the client now uses the TLS endpoints and requires the user to specify a trust store for validating the server certificate. -* Updated to Quarkus 3.2.6.Final +* All components now use Quarkus 3.2.6.Final. +* The JDBC based Device Registry implementation now has limited support for filter criteria when searching devices. + Please refer to the Device Registry User Guide for details. +* The Device Registry Management API's *search Devices* operation now supports restricting the result set to gateway or + edge devices only using the newly added `isGateway` query parameter. ### Deprecations diff --git a/tests/src/test/java/org/eclipse/hono/tests/DeviceRegistryHttpClient.java b/tests/src/test/java/org/eclipse/hono/tests/DeviceRegistryHttpClient.java index 1d06caafe6..cb736202b8 100644 --- a/tests/src/test/java/org/eclipse/hono/tests/DeviceRegistryHttpClient.java +++ b/tests/src/test/java/org/eclipse/hono/tests/DeviceRegistryHttpClient.java @@ -783,6 +783,8 @@ public Future> deregisterDevice( * @param pageOffset The offset into the result set from which to include objects in the response. * @param filters The filters are predicates that objects in the result set must match. * @param sortOptions A list of sort options. + * @param isGateway A filter for restricting the search to gateway ({@code True}) or edge ({@code False} devices only. + * If empty, the search will not be restricted. * @param expectedStatusCode The status code indicating a successful outcome. * @return A future indicating the outcome of the operation. The future will contain the response if the * response contained the expected status code. Otherwise the future will fail. @@ -794,6 +796,7 @@ public Future> searchDevices( final Optional pageOffset, final List filters, final List sortOptions, + final Optional isGateway, final int expectedStatusCode) { Objects.requireNonNull(tenantId); @@ -811,6 +814,7 @@ public Future> searchDevices( pOffset -> queryParams.add(RegistryManagementConstants.PARAM_PAGE_OFFSET, String.valueOf(pOffset))); filters.forEach(filterJson -> queryParams.add(RegistryManagementConstants.PARAM_FILTER_JSON, filterJson)); sortOptions.forEach(sortJson -> queryParams.add(RegistryManagementConstants.PARAM_SORT_JSON, sortJson)); + isGateway.ifPresent(b -> queryParams.add(RegistryManagementConstants.PARAM_IS_GATEWAY, b.toString())); return httpClient.get(requestUri, getRequestHeaders(), queryParams, ResponsePredicate.status(expectedStatusCode)); } diff --git a/tests/src/test/java/org/eclipse/hono/tests/registry/DeviceManagementIT.java b/tests/src/test/java/org/eclipse/hono/tests/registry/DeviceManagementIT.java index e8f3bdd9c8..a9910e7901 100644 --- a/tests/src/test/java/org/eclipse/hono/tests/registry/DeviceManagementIT.java +++ b/tests/src/test/java/org/eclipse/hono/tests/registry/DeviceManagementIT.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2016, 2023 Contributors to the Eclipse Foundation + * Copyright (c) 2016 Contributors to the Eclipse Foundation * * See the NOTICE file(s) distributed with this work for additional * information regarding copyright ownership. @@ -594,8 +594,14 @@ public void testSearchDevicesFailsWhenNoDevicesAreFound(final VertxTestContext c final String filterJson = getFilterJson("/enabled", true, "eq"); registry.registerDevice(tenantId, deviceId, device) - .compose(ok -> registry.searchDevices(tenantId, Optional.empty(), Optional.empty(), - List.of(filterJson), List.of(), HttpURLConnection.HTTP_NOT_FOUND)) + .compose(ok -> registry.searchDevices( + tenantId, + Optional.empty(), + Optional.empty(), + List.of(filterJson), + List.of(), + Optional.empty(), + HttpURLConnection.HTTP_NOT_FOUND)) .onComplete(ctx.succeedingThenComplete()); } @@ -609,8 +615,14 @@ public void testSearchDevicesWithInvalidPageSizeFails(final VertxTestContext ctx final int invalidPageSize = -100; registry.registerDevice(tenantId, deviceId) - .compose(ok -> registry.searchDevices(tenantId, Optional.of(invalidPageSize), Optional.empty(), - List.of(), List.of(), HttpURLConnection.HTTP_BAD_REQUEST)) + .compose(ok -> registry.searchDevices( + tenantId, + Optional.of(invalidPageSize), + Optional.empty(), + List.of(), + List.of(), + Optional.empty(), + HttpURLConnection.HTTP_BAD_REQUEST)) .onComplete(ctx.succeedingThenComplete()); } @@ -634,6 +646,7 @@ public void testSearchDevicesWithValidPageSizeSucceeds(final VertxTestContext ct Optional.empty(), List.of(), List.of(), + Optional.empty(), HttpURLConnection.HTTP_OK)) .onComplete(ctx.succeeding(httpResponse -> { ctx.verify(() -> { @@ -656,8 +669,14 @@ public void testSearchDevicesWithInvalidPageOffsetFails(final VertxTestContext c final int invalidPageOffset = -100; registry.registerDevice(tenantId, deviceId) - .compose(ok -> registry.searchDevices(tenantId, Optional.empty(), Optional.of(invalidPageOffset), - List.of(), List.of(), HttpURLConnection.HTTP_BAD_REQUEST)) + .compose(ok -> registry.searchDevices( + tenantId, + Optional.empty(), + Optional.of(invalidPageOffset), + List.of(), + List.of(), + Optional.empty(), + HttpURLConnection.HTTP_BAD_REQUEST)) .onComplete(ctx.succeedingThenComplete()); } @@ -680,8 +699,14 @@ public void testSearchDevicesWithValidPageOffsetSucceeds(final VertxTestContext Future.all( registry.registerDevice(tenantId, deviceId1, device1), registry.registerDevice(tenantId, deviceId2, device2)) - .compose(ok -> registry.searchDevices(tenantId, Optional.of(pageSize), Optional.of(pageOffset), - List.of(), List.of(sortJson), HttpURLConnection.HTTP_OK)) + .compose(ok -> registry.searchDevices( + tenantId, + Optional.of(pageSize), + Optional.of(pageOffset), + List.of(), + List.of(sortJson), + Optional.empty(), + HttpURLConnection.HTTP_OK)) .onComplete(ctx.succeeding(httpResponse -> { ctx.verify(() -> { final SearchResult searchDevicesResult = JacksonCodec @@ -703,8 +728,14 @@ public void testSearchDevicesWithValidPageOffsetSucceeds(final VertxTestContext public void testSearchDevicesWithInvalidFilterJsonFails(final VertxTestContext ctx) { registry.registerDevice(tenantId, deviceId) - .compose(ok -> registry.searchDevices(tenantId, Optional.empty(), Optional.empty(), - List.of("Invalid filterJson"), List.of(), HttpURLConnection.HTTP_BAD_REQUEST)) + .compose(ok -> registry.searchDevices( + tenantId, + Optional.empty(), + Optional.empty(), + List.of("Invalid filterJson"), + List.of(), + Optional.empty(), + HttpURLConnection.HTTP_BAD_REQUEST)) .onComplete(ctx.succeedingThenComplete()); } @@ -726,10 +757,22 @@ public void testSearchDevicesWithValidMultipleFiltersSucceeds(final VertxTestCon Future.all( registry.registerDevice(tenantId, deviceId1, device1), registry.registerDevice(tenantId, deviceId2, device2)) - .compose(ok -> registry.searchDevices(tenantId, Optional.empty(), Optional.empty(), - List.of(filterJson1, filterJson2), List.of(), HttpURLConnection.HTTP_NOT_FOUND)) - .compose(ok -> registry.searchDevices(tenantId, Optional.empty(), Optional.empty(), - List.of(filterJson1, filterJson3), List.of(), HttpURLConnection.HTTP_OK)) + .compose(ok -> registry.searchDevices( + tenantId, + Optional.empty(), + Optional.empty(), + List.of(filterJson1, filterJson2), + List.of(), + Optional.empty(), + HttpURLConnection.HTTP_NOT_FOUND)) + .compose(ok -> registry.searchDevices( + tenantId, + Optional.empty(), + Optional.empty(), + List.of(filterJson1, filterJson3), + List.of(), + Optional.empty(), + HttpURLConnection.HTTP_OK)) .onComplete(ctx.succeeding(httpResponse -> { ctx.verify(() -> { final SearchResult searchDevicesResult = JacksonCodec @@ -760,8 +803,14 @@ public void testSearchDevicesWithWildCardToMatchMultipleCharactersSucceeds(final Future.all( registry.registerDevice(tenantId, deviceId1, device1), registry.registerDevice(tenantId, deviceId2, device2)) - .compose(ok -> registry.searchDevices(tenantId, Optional.empty(), Optional.empty(), - List.of(filterJson1, filterJson2), List.of(), HttpURLConnection.HTTP_OK)) + .compose(ok -> registry.searchDevices( + tenantId, + Optional.empty(), + Optional.empty(), + List.of(filterJson1, filterJson2), + List.of(), + Optional.empty(), + HttpURLConnection.HTTP_OK)) .onComplete(ctx.succeeding(httpResponse -> { ctx.verify(() -> { final SearchResult searchDevicesResult = JacksonCodec @@ -774,6 +823,45 @@ public void testSearchDevicesWithWildCardToMatchMultipleCharactersSucceeds(final })); } + /** + * Verifies that a request to search gateway devices succeeds and matching devices are found. + * + * @param ctx The vert.x test context. + */ + @Test + public void testSearchGatewayDevicesSucceeds(final VertxTestContext ctx) { + final String deviceId1 = "1_" + getHelper().getRandomDeviceId(tenantId); + final String deviceId2 = "2_" + getHelper().getRandomDeviceId(tenantId); + final String deviceId3 = "3_" + getHelper().getRandomDeviceId(tenantId); + final Device device1 = new Device().setMemberOf(List.of("gwGroup1")); + final Device device2 = new Device().setVia(List.of(deviceId3)); + final Device device3 = new Device(); + + Future.all( + registry.registerDevice(tenantId, deviceId1, device1), + registry.registerDevice(tenantId, deviceId2, device2), + registry.registerDevice(tenantId, deviceId3, device3)) + .compose(ok -> registry.searchDevices( + tenantId, + Optional.empty(), + Optional.empty(), + List.of(), + List.of(), + Optional.of(true), + HttpURLConnection.HTTP_OK)) + .onComplete(ctx.succeeding(httpResponse -> { + ctx.verify(() -> { + final SearchResult searchDevicesResult = JacksonCodec + .decodeValue(httpResponse.body(), new TypeReference<>() { }); + assertThat(searchDevicesResult.getTotal()).isEqualTo(2); + assertThat(searchDevicesResult.getResult()).hasSize(2); + assertThat(searchDevicesResult.getResult().get(0).getId()).isEqualTo(deviceId1); + assertThat(searchDevicesResult.getResult().get(1).getId()).isEqualTo(deviceId3); + }); + ctx.completeNow(); + })); + } + /** * Verifies that a request to search devices with a filter containing the wildcard character '*' fails with a * {@value HttpURLConnection#HTTP_NOT_FOUND} as no matching devices are found. @@ -787,8 +875,14 @@ public void testSearchDevicesWithWildCardToMatchMultipleCharactersFails(final Ve final String filterJson = getFilterJson("/ext/id", "*id*2", "eq"); registry.registerDevice(tenantId, deviceId, device) - .compose(ok -> registry.searchDevices(tenantId, Optional.empty(), Optional.empty(), - List.of(filterJson), List.of(), HttpURLConnection.HTTP_NOT_FOUND)) + .compose(ok -> registry.searchDevices( + tenantId, + Optional.empty(), + Optional.empty(), + List.of(filterJson), + List.of(), + Optional.empty(), + HttpURLConnection.HTTP_NOT_FOUND)) .onComplete(ctx.succeedingThenComplete()); } @@ -810,8 +904,14 @@ public void testSearchDevicesWithWildCardToMatchExactlyOneCharacterSucceeds(fina Future.all( registry.registerDevice(tenantId, deviceId1, device1), registry.registerDevice(tenantId, deviceId2, device2)) - .compose(ok -> registry.searchDevices(tenantId, Optional.empty(), Optional.empty(), - List.of(filterJson1, filterJson2), List.of(), HttpURLConnection.HTTP_OK)) + .compose(ok -> registry.searchDevices( + tenantId, + Optional.empty(), + Optional.empty(), + List.of(filterJson1, filterJson2), + List.of(), + Optional.empty(), + HttpURLConnection.HTTP_OK)) .onComplete(ctx.succeeding(httpResponse -> { ctx.verify(() -> { final SearchResult searchDevicesResult = JacksonCodec @@ -837,8 +937,14 @@ public void testSearchDevicesWithWildCardToMatchExactlyOneCharacterFails(final V final String filterJson = getFilterJson("/ext/id", "$id:?2", "eq"); registry.registerDevice(tenantId, deviceId, device) - .compose(ok -> registry.searchDevices(tenantId, Optional.empty(), Optional.empty(), - List.of(filterJson), List.of(), HttpURLConnection.HTTP_NOT_FOUND)) + .compose(ok -> registry.searchDevices( + tenantId, + Optional.empty(), + Optional.empty(), + List.of(filterJson), + List.of(), + Optional.empty(), + HttpURLConnection.HTTP_NOT_FOUND)) .onComplete(ctx.succeedingThenComplete()); } @@ -851,8 +957,14 @@ public void testSearchDevicesWithWildCardToMatchExactlyOneCharacterFails(final V public void testSearchDevicesWithInvalidSortJsonFails(final VertxTestContext ctx) { registry.registerDevice(tenantId, deviceId) - .compose(ok -> registry.searchDevices(tenantId, Optional.empty(), Optional.empty(), List.of(), - List.of("Invalid sortJson"), HttpURLConnection.HTTP_BAD_REQUEST)) + .compose(ok -> registry.searchDevices( + tenantId, + Optional.empty(), + Optional.empty(), + List.of(), + List.of("Invalid sortJson"), + Optional.empty(), + HttpURLConnection.HTTP_BAD_REQUEST)) .onComplete(ctx.succeedingThenComplete()); } @@ -873,8 +985,14 @@ public void testSearchDevicesWithValidSortOptionSucceeds(final VertxTestContext Future.all( registry.registerDevice(tenantId, deviceId1, device1), registry.registerDevice(tenantId, deviceId2, device2)) - .compose(ok -> registry.searchDevices(tenantId, Optional.empty(), Optional.empty(), List.of(), - List.of(sortJson), HttpURLConnection.HTTP_OK)) + .compose(ok -> registry.searchDevices( + tenantId, + Optional.empty(), + Optional.empty(), + List.of(), + List.of(sortJson), + Optional.empty(), + HttpURLConnection.HTTP_OK)) .onComplete(ctx.succeeding(httpResponse -> { ctx.verify(() -> { final SearchResult searchDevicesResult = JacksonCodec