Support models repo metadata functionality and eliminate TryFromExpanded from API surface

sdk/modelsrepository/azure-iot-modelsrepository/src/main/java/com/azure/iot/modelsrepository/DtmiConventions.java

@@ -74,6 +74,26 @@ public static URI getModelUri(String dtmi, URI repositoryUri, boolean expanded)
         }
     }
 
+    /**
+     * Generates the model repository's metadata URI.
+     *
+     * @param repositoryUri The repository uri
+     * @return The repository metadata uri.
+     * @throws IllegalArgumentException if the provided repository URI is not valid
+     */
+    public static URI getMetadataUri(URI repositoryUri) {
+        try {
+            String stringUri = repositoryUri.toString();
+            if (stringUri.endsWith("/")) {
+                return new URI(stringUri + ModelsRepositoryConstants.MODELS_REPOSITORY_METADATA_FILE);
+            } else {
+                return new URI(stringUri + "/" + ModelsRepositoryConstants.MODELS_REPOSITORY_METADATA_FILE);
+            }
+        } catch (URISyntaxException e) {
+            throw new IllegalArgumentException("Invalid uri syntax");
+        }
+    }
+
     static String dtmiToPath(String dtmi) {
         if (!isValidDtmi(dtmi)) {
             throw new IllegalArgumentException(String.format(StatusStrings.INVALID_DTMI_FORMAT_S, dtmi));

sdk/modelsrepository/azure-iot-modelsrepository/src/main/java/com/azure/iot/modelsrepository/ModelDependencyResolution.java

@@ -16,10 +16,4 @@ public enum ModelDependencyResolution {
      * Enable model dependency resolution. The client will parse models and calculate dependencies recursively.
      */
     ENABLED,
-
-    /**
-     * Try to get pre-computed model dependencies using .expanded.json.
-     * If the model expanded form does not exist, it will fall back to {@link ModelDependencyResolution#ENABLED}.
-     */
-    TRY_FROM_EXPANDED,
 }

sdk/modelsrepository/azure-iot-modelsrepository/src/main/java/com/azure/iot/modelsrepository/ModelsRepositoryClientBuilder.java

@@ -58,7 +58,7 @@ public final class ModelsRepositoryClientBuilder {
     // Fields with default values.
     private URI repositoryEndpoint;
 
-    private ModelDependencyResolution modelDependencyResolution = ModelDependencyResolution.TRY_FROM_EXPANDED;
+    private ModelDependencyResolution modelDependencyResolution = ModelDependencyResolution.ENABLED;
 
     // optional/have default values
     private ModelsRepositoryServiceVersion serviceVersion;
@@ -152,7 +152,7 @@ private static HttpPipeline constructPipeline(
     /**
      * Create a {@link ModelsRepositoryClient} based on the builder settings.
      *
-     * @return the created synchronous ModelsRepotioryClient
+     * @return the created synchronous ModelsRepositoryClient
      */
     public ModelsRepositoryClient buildClient() {
         return new ModelsRepositoryClient(buildAsyncClient());

sdk/modelsrepository/azure-iot-modelsrepository/src/main/java/com/azure/iot/modelsrepository/implementation/FileModelFetcher.java

@@ -7,8 +7,8 @@
 import com.azure.core.util.Context;
 import com.azure.core.util.logging.ClientLogger;
 import com.azure.iot.modelsrepository.DtmiConventions;
-import com.azure.iot.modelsrepository.ModelDependencyResolution;
-import com.azure.iot.modelsrepository.implementation.models.FetchResult;
+import com.azure.iot.modelsrepository.implementation.models.FetchMetadataResult;
+import com.azure.iot.modelsrepository.implementation.models.FetchModelResult;
 import reactor.core.publisher.Mono;
 
 import java.io.File;
@@ -39,15 +39,15 @@ class FileModelFetcher implements ModelFetcher {
     }
 
     @Override
-    public Mono<FetchResult> fetchAsync(String dtmi, URI repositoryUri, ModelDependencyResolution resolutionOption, Context context) {
+    public Mono<FetchModelResult> fetchModelAsync(String dtmi, URI repositoryUri, boolean tryFromExpanded, Context context) {
         return Mono.defer(() -> {
             Queue<String> work = new LinkedList<>();
 
             try {
-                if (resolutionOption == ModelDependencyResolution.TRY_FROM_EXPANDED) {
-                    work.add(getPath(dtmi, repositoryUri, true));
+                if (tryFromExpanded) {
+                    work.add(getModelPath(dtmi, repositoryUri, true));
                 }
-                work.add(getPath(dtmi, repositoryUri, false));
+                work.add(getModelPath(dtmi, repositoryUri, false));
             } catch (MalformedURLException | URISyntaxException e) {
                 return Mono.error(new AzureException(e));
             }
@@ -63,7 +63,7 @@ public Mono<FetchResult> fetchAsync(String dtmi, URI repositoryUri, ModelDepende
                 if (Files.exists(path)) {
                     try {
                         return Mono.just(
-                            new FetchResult()
+                            new FetchModelResult()
                                 .setDefinition(new String(Files.readAllBytes(path), StandardCharsets.UTF_8))
                                 .setPath(tryContentPath));
                     } catch (IOException e) {
@@ -80,8 +80,52 @@ public Mono<FetchResult> fetchAsync(String dtmi, URI repositoryUri, ModelDepende
         });
     }
 
-    private String getPath(String dtmi, URI repositoryUri, boolean expanded) throws URISyntaxException, MalformedURLException {
+    @Override
+    public Mono<FetchMetadataResult> fetchMetadataAsync(URI repositoryUri, Context context) {
+        return Mono.defer(() -> {
+            Queue<String> work = new LinkedList<>();
+
+            try {
+                work.add(getMetadataPath(repositoryUri));
+            } catch (MalformedURLException | URISyntaxException e) {
+                return Mono.error(new AzureException(e));
+            }
+
+            String fnfError = "";
+            while (work.size() != 0) {
+                String tryContentPath = work.poll();
+
+                Path path = Paths.get(new File(tryContentPath).getPath());
+
+                logger.info(StatusStrings.FETCHING_METADATA_CONTENT, path);
+
+                if (Files.exists(path)) {
+                    try {
+                        return Mono.just(
+                            new FetchMetadataResult()
+                                .setDefinition(new String(Files.readAllBytes(path), StandardCharsets.UTF_8))
+                                .setPath(tryContentPath));
+                    } catch (IOException e) {
+                        return Mono.error(new AzureException(e));
+                    }
+                }
+
+                logger.error(String.format(StatusStrings.ERROR_FETCHING_METADATA_CONTENT, path.toString()));
+
+                fnfError = String.format(StatusStrings.ERROR_FETCHING_METADATA_CONTENT, tryContentPath);
+            }
+
+            return Mono.error(new AzureException(fnfError));
+        });
+    }
+
+    private String getModelPath(String dtmi, URI repositoryUri, boolean expanded) throws URISyntaxException, MalformedURLException {
         return DtmiConventions.getModelUri(dtmi, repositoryUri, expanded)
             .getPath();
     }
+
+    private String getMetadataPath(URI repositoryUri) throws URISyntaxException, MalformedURLException {
+        return DtmiConventions.getMetadataUri(repositoryUri)
+            .getPath();
+    }
 }

sdk/modelsrepository/azure-iot-modelsrepository/src/main/java/com/azure/iot/modelsrepository/implementation/HttpModelFetcher.java

@@ -7,10 +7,12 @@
 import com.azure.core.util.Context;
 import com.azure.core.util.logging.ClientLogger;
 import com.azure.iot.modelsrepository.DtmiConventions;
-import com.azure.iot.modelsrepository.ModelDependencyResolution;
-import com.azure.iot.modelsrepository.implementation.models.FetchResult;
+import com.azure.iot.modelsrepository.implementation.models.FetchMetadataResult;
+import com.azure.iot.modelsrepository.implementation.models.FetchModelResult;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import reactor.core.publisher.Mono;
 
+import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
@@ -31,14 +33,14 @@ class HttpModelFetcher implements ModelFetcher {
     }
 
     @Override
-    public Mono<FetchResult> fetchAsync(String dtmi, URI repositoryUri, ModelDependencyResolution resolutionOption, Context context) {
+    public Mono<FetchModelResult> fetchModelAsync(String dtmi, URI repositoryUri, boolean tryFromExpanded, Context context) {
         return Mono.defer(() -> {
             Queue<String> work = new LinkedList<>();
             try {
-                if (resolutionOption == ModelDependencyResolution.TRY_FROM_EXPANDED) {
-                    work.add(getPath(dtmi, repositoryUri, true));
+                if (tryFromExpanded) {
+                    work.add(getModelPath(dtmi, repositoryUri, true));
                 }
-                work.add(getPath(dtmi, repositoryUri, false));
+                work.add(getModelPath(dtmi, repositoryUri, false));
             } catch (Exception e) {
                 return Mono.error(new AzureException(e));
             }
@@ -52,10 +54,45 @@ public Mono<FetchResult> fetchAsync(String dtmi, URI repositoryUri, ModelDepende
                     if (work.size() != 0) {
                         return evaluatePath(work.poll(), context);
                     } else {
+                        logger.error(String.format(StatusStrings.ERROR_FETCHING_MODEL_CONTENT, tryContentPath));
                         return Mono.error(error);
                     }
                 })
-                .map(s -> new FetchResult().setPath(tryContentPath).setDefinition(s));
+                .map(s -> new FetchModelResult().setPath(tryContentPath).setDefinition(s));
+        });
+    }
+
+    @Override
+    public Mono<FetchMetadataResult> fetchMetadataAsync(URI repositoryUri, Context context) {
+        return Mono.defer(() -> {
+            Queue<String> work = new LinkedList<>();
+            try {
+                work.add(getMetadataPath(repositoryUri));
+            } catch (Exception e) {
+                return Mono.error(new AzureException(e));
+            }
+
+            String tryContentPath = work.poll();
+
+            logger.info(StatusStrings.FETCHING_METADATA_CONTENT, tryContentPath);
+
+            return evaluatePath(tryContentPath, context)
+                .onErrorResume(error -> {
+                    if (work.size() != 0) {
+                        return evaluatePath(work.poll(), context);
+                    } else {
+                        logger.error(String.format(StatusStrings.ERROR_FETCHING_METADATA_CONTENT, tryContentPath));
+                        return Mono.error(error);
+                    }
+                })
+                .map(s -> {
+                    try {
+                        return new FetchMetadataResult().setPath(tryContentPath).setDefinition(s);
+                    } catch (JsonProcessingException e) {
+                        logger.error(String.format(StatusStrings.ERROR_FETCHING_METADATA_CONTENT, tryContentPath));
+                        return null;
+                    }
+                });
         });
     }
 
@@ -69,7 +106,11 @@ private Mono<String> evaluatePath(String tryContentPath, Context context) {
             });
     }
 
-    private String getPath(String dtmi, URI repositoryUri, boolean expanded) throws URISyntaxException {
+    private String getModelPath(String dtmi, URI repositoryUri, boolean expanded) throws URISyntaxException {
         return DtmiConventions.getModelUri(dtmi, repositoryUri, expanded).getPath();
     }
+
+    private String getMetadataPath(URI repositoryUri) throws URISyntaxException, MalformedURLException {
+        return DtmiConventions.getMetadataUri(repositoryUri).getPath();
+    }
 }

sdk/modelsrepository/azure-iot-modelsrepository/src/main/java/com/azure/iot/modelsrepository/implementation/IntermediateFetchResult.java → sdk/modelsrepository/azure-iot-modelsrepository/src/main/java/com/azure/iot/modelsrepository/implementation/IntermediateFetchModelResult.java

@@ -3,25 +3,25 @@
 
 package com.azure.iot.modelsrepository.implementation;
 
-import com.azure.iot.modelsrepository.implementation.models.FetchResult;
+import com.azure.iot.modelsrepository.implementation.models.FetchModelResult;
 
 import java.util.Map;
 
 /**
  * This type is used to unify the expand operation return types in the recursive function and has no other use cases.
  * Do not take any dependencies on this type.
  */
-class IntermediateFetchResult {
-    private final FetchResult fetchResult;
+class IntermediateFetchModelResult {
+    private final FetchModelResult fetchModelResult;
     private final Map<String, String> map;
 
-    IntermediateFetchResult(FetchResult fetchResult, Map<String, String> map) {
-        this.fetchResult = fetchResult;
+    IntermediateFetchModelResult(FetchModelResult fetchModelResult, Map<String, String> map) {
+        this.fetchModelResult = fetchModelResult;
         this.map = map;
     }
 
-    public FetchResult getFetchResult() {
-        return fetchResult;
+    public FetchModelResult getFetchModelResult() {
+        return fetchModelResult;
     }
 
     public Map<String, String> getMap() {

sdk/modelsrepository/azure-iot-modelsrepository/src/main/java/com/azure/iot/modelsrepository/implementation/ModelFetcher.java

@@ -4,12 +4,13 @@
 package com.azure.iot.modelsrepository.implementation;
 
 import com.azure.core.util.Context;
-import com.azure.iot.modelsrepository.ModelDependencyResolution;
-import com.azure.iot.modelsrepository.implementation.models.FetchResult;
+import com.azure.iot.modelsrepository.implementation.models.FetchMetadataResult;
+import com.azure.iot.modelsrepository.implementation.models.FetchModelResult;
 import reactor.core.publisher.Mono;
 
 import java.net.URI;
 
 interface ModelFetcher {
-    Mono<FetchResult> fetchAsync(String dtmi, URI repositoryUri, ModelDependencyResolution resolutionOption, Context context);
+    Mono<FetchModelResult> fetchModelAsync(String dtmi, URI repositoryUri, boolean tryFromExpanded, Context context);
+    Mono<FetchMetadataResult> fetchMetadataAsync(URI repositoryUri, Context context);
 }

sdk/modelsrepository/azure-iot-modelsrepository/src/main/java/com/azure/iot/modelsrepository/implementation/ModelsRepositoryConstants.java

@@ -9,6 +9,7 @@ public class ModelsRepositoryConstants {
     public static final String JSON_EXTENSION = ".json";
     public static final String JSON_EXPANDED_EXTENSION = ".expanded.json";
     public static final String DEFAULT_MODELS_REPOSITORY_ENDPOINT = "https://devicemodels.azure.com";
+    public static final String MODELS_REPOSITORY_METADATA_FILE = "metadata.json";
 
     // DTDL conventions
     public static final String DTDL_TYPE = "@type";

sdk/modelsrepository/azure-iot-modelsrepository/src/main/java/com/azure/iot/modelsrepository/implementation/RepositoryHandler.java

@@ -8,7 +8,8 @@
 import com.azure.core.util.logging.ClientLogger;
 import com.azure.iot.modelsrepository.DtmiConventions;
 import com.azure.iot.modelsrepository.ModelDependencyResolution;
-import com.azure.iot.modelsrepository.implementation.models.FetchResult;
+import com.azure.iot.modelsrepository.implementation.models.FetchMetadataResult;
+import com.azure.iot.modelsrepository.implementation.models.FetchModelResult;
 import com.azure.iot.modelsrepository.implementation.models.ModelMetadata;
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
@@ -55,10 +56,10 @@ public Mono<Map<String, String>> processAsync(Iterable<String> dtmis, ModelDepen
 
         return processAsync(modelsToProcess, resolutionOptions, context, processedModels)
             .last()
-            .map(IntermediateFetchResult::getMap);
+            .map(IntermediateFetchModelResult::getMap);
     }
 
-    private Flux<IntermediateFetchResult> processAsync(
+    private Flux<IntermediateFetchModelResult> processAsync(
         Queue<String> remainingWork,
         ModelDependencyResolution resolutionOption,
         Context context,
@@ -69,14 +70,39 @@ private Flux<IntermediateFetchResult> processAsync(
         }
 
         String targetDtmi = remainingWork.poll();
+        Boolean tryFromExpanded = false;
+
+        // If ModelDependencyResolution.Enabled is requested the client will first attempt to fetch
+        // metadata.json content from the target repository. The metadata object includes supported features
+        // of the repository.
+        // If the metadata indicates expanded models are available. The client will try to fetch pre-computed model
+        // dependencies using .expanded.json.
+        // If the model expanded form does not exist fall back to computing model dependencies just-in-time.
+        if (resolutionOption == ModelDependencyResolution.ENABLED) {
+            Mono<FetchMetadataResult> repositoryMetadata = modelFetcher.fetchMetadataAsync(repositoryUri, context);
+
+            if (repositoryMetadata != null) {
+                tryFromExpanded = repositoryMetadata
+                    .flatMap(repo -> {
+                        if (repo != null && repo.getDefinition() != null
+                            && repo.getDefinition().getFeatures() != null
+                            && repo.getDefinition().getFeatures().isExpanded()) {
+                            return Mono.just(true);
+                        }
+                        return Mono.just(false);
+                    })
+                    .defaultIfEmpty(false)
+                    .block();
+            }
+        }
 
         logger.info(String.format(StatusStrings.PROCESSING_DTMIS, targetDtmi));
 
-        return modelFetcher.fetchAsync(targetDtmi, repositoryUri, resolutionOption, context)
-            .map(result -> new IntermediateFetchResult(result, currentResults))
+        return modelFetcher.fetchModelAsync(targetDtmi, repositoryUri, tryFromExpanded, context)
+            .map(result -> new IntermediateFetchModelResult(result, currentResults))
             .expand(customType -> {
                 Map<String, String> results = customType.getMap();
-                FetchResult response = customType.getFetchResult();
+                FetchModelResult response = customType.getFetchModelResult();
 
                 if (response.isFromExpanded()) {
                     try {
@@ -96,7 +122,7 @@ private Flux<IntermediateFetchResult> processAsync(
                 try {
                     ModelMetadata metadata = new ModelsQuery(response.getDefinition()).parseModel();
 
-                    if (resolutionOption == ModelDependencyResolution.ENABLED || resolutionOption == ModelDependencyResolution.TRY_FROM_EXPANDED) {
+                    if (resolutionOption == ModelDependencyResolution.ENABLED) {
                         List<String> dependencies = metadata.getDependencies();
 
                         if (dependencies.size() > 0) {

sdk/modelsrepository/azure-iot-modelsrepository/src/main/java/com/azure/iot/modelsrepository/implementation/StatusStrings.java

@@ -9,6 +9,8 @@ public class StatusStrings {
     public static final String PROCESSING_DTMIS = "Processing DTMI \"%s\". ";
     public static final String DISCOVERED_DEPENDENCIES = "Discovered dependencies \"%s\".";
     public static final String FETCHING_MODEL_CONTENT = "Attempting to fetch model content from \"{}\".";
+    public static final String FETCHING_METADATA_CONTENT = "Attempting to fetch repository metadata content from \"{}\".";
     public static final String ERROR_FETCHING_MODEL_CONTENT = "Model file \"%s\" not found or not accessible in target repository.";
+    public static final String ERROR_FETCHING_METADATA_CONTENT = "Metadata file \"%s\" not found or not accessible in target repository.";
     public static final String INCORRECT_DTMI_CASING = "Fetched model has incorrect DTMI casing. Expected \"%s\", parsed \"%s\".";
 }