Skip to content

Commit

Permalink
Support for dynamic loading of provider definitions (#10868)
Browse files Browse the repository at this point in the history
## Overview
Adds support for loading the 'az' provider dynamically sourced from an
OCI artifact registry. The artifact must be restored (downloaded to the
appropriate location) manually under

`$BicepCacheRootDir/br/mcr.microsoft.com/bicep$providers$az/${providerVersion}`
which is the target location in the Bicep cache for the `az` provider.

The work to restore the provider data to the cache will be handled in a
separate PR for convenience so that reviewing PRs is easier.

To enable the feature `DynamicTypeLoadingEnabled` and must be set to
true in `bicepconfig.json`:
```json
{
    "experimentalFeaturesEnabled": {
        "extensibility": true,
        "dynamicTypeLoadingEnabled": true,
    },
    "cacheRootDirectory": "~/.bicep",
}
```
## Changes
- Adds a new experimental feature flag `DynamicTypeLoadingEnabled`
- Adds a new factory `AzResourceTypeLoader` that is handling the concern
of deciding the provider loader to use based on the feature flag and
presence of an import declaration syntax in the Bicep file being
processed.
- Serializes the provider version by inspecting the
`ImportDeclarationSyntax` vs using a hardcoded value

Fixes #10662

## Contributing a feature

* [x] I have opened a new issue for the proposal, or commented on an
existing one, and ensured that the Bicep maintainers are good with the
design of the feature being implemented
* [x] I have included "Fixes #{issue_number}" in the PR description, so
GitHub can link to the issue and close it when the PR is merged
* [x] I have appropriate test coverage of my new feature

---------

Co-authored-by: Ariel Silverman <[email protected]>
  • Loading branch information
asilverman and Ariel Silverman authored Jul 20, 2023
1 parent 9da3e4c commit b939e3d
Show file tree
Hide file tree
Showing 60 changed files with 575 additions and 320 deletions.
8 changes: 5 additions & 3 deletions .github/workflows/codecov.runsettings
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
<?xml version="1.0" encoding="utf-8" ?>
<?xml version="1.0" encoding="utf-8"?>
<RunSettings>
<RunConfiguration>
<!-- Break builds when no tests are discovered. -->
<!-- Add a total session timeout of 30min -->
<TestSessionTimeout>1800000 </TestSessionTimeout>
<TreatNoTestsAsError>false</TreatNoTestsAsError>
</RunConfiguration>
<DataCollectionRunSettings>
<DataCollectors>
<!-- "Code Coverage" settings documented in https://learn.microsoft.com/en-us/visualstudio/test/customizing-code-coverage-analysis?view=vs-2022 -->
<!-- "Code Coverage" settings documented in
https://learn.microsoft.com/en-us/visualstudio/test/customizing-code-coverage-analysis?view=vs-2022 -->
<DataCollector friendlyName="Code Coverage">
<Configuration>
<Format>cobertura</Format>
Expand Down
6 changes: 1 addition & 5 deletions src/Bicep.Cli/Helpers/ServiceCollectionExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
using Bicep.Core;
using Bicep.Core.Analyzers.Interfaces;
using Bicep.Core.Analyzers.Linter;
using Bicep.Core.Analyzers.Linter.ApiVersions;
using Bicep.Core.Configuration;
using Bicep.Core.Features;
using Bicep.Core.FileSystem;
Expand All @@ -15,10 +14,7 @@
using Bicep.Core.TypeSystem.Az;
using Bicep.Decompiler;
using Microsoft.Extensions.DependencyInjection;
using System;
using System.Collections.Generic;
using System.IO.Abstractions;
using System.Linq;
using IOFileSystem = System.IO.Abstractions.FileSystem;

namespace Bicep.Cli.Helpers;
Expand Down Expand Up @@ -57,6 +53,7 @@ public static IServiceCollection AddCommands(this IServiceCollection services) =
public static IServiceCollection AddBicepCore(this IServiceCollection services) => services
.AddSingleton<INamespaceProvider, DefaultNamespaceProvider>()
.AddSingleton<IAzResourceTypeLoader, AzResourceTypeLoader>()
.AddSingleton<IAzResourceTypeLoaderFactory, AzResourceTypeLoaderFactory>()
.AddSingleton<IContainerRegistryClientFactory, ContainerRegistryClientFactory>()
.AddSingleton<ITemplateSpecRepositoryFactory, TemplateSpecRepositoryFactory>()
.AddSingleton<IModuleDispatcher, ModuleDispatcher>()
Expand All @@ -65,7 +62,6 @@ public static IServiceCollection AddBicepCore(this IServiceCollection services)
.AddSingleton<IFileResolver, FileResolver>()
.AddSingleton<IFileSystem, IOFileSystem>()
.AddSingleton<IConfigurationManager, ConfigurationManager>()
.AddSingleton<IApiVersionProviderFactory, ApiVersionProviderFactory>()
.AddSingleton<IBicepAnalyzer, LinterAnalyzer>()
.AddSingleton<IFeatureProviderFactory, FeatureProviderFactory>()
.AddSingleton<ILinterRulesProvider, LinterRulesProvider>()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ public bool HasDefinedType(ResourceTypeReference typeReference)

public IEnumerable<ResourceTypeReference> GetAvailableTypes()
=> resourceTypes.Keys;

public string Version { get; } = "1.0.0";
}

public static NamespaceType Create(string aliasName)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,8 @@ public bool HasDefinedType(ResourceTypeReference typeReference)

public IEnumerable<ResourceTypeReference> GetAvailableTypes()
=> resourceTypes.Keys;

public string Version { get; } = "1.0.0";
}

public static NamespaceType Create(string aliasName)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,23 @@ public class TestExtensibilityNamespaceProvider : INamespaceProvider
{
private readonly INamespaceProvider defaultNamespaceProvider;

public TestExtensibilityNamespaceProvider(IAzResourceTypeLoader azResourceTypeLoader)
public TestExtensibilityNamespaceProvider(IAzResourceTypeLoaderFactory azResourceTypeLoaderFactory)
{
defaultNamespaceProvider = new DefaultNamespaceProvider(azResourceTypeLoader);
defaultNamespaceProvider = new DefaultNamespaceProvider(azResourceTypeLoaderFactory);
}

public IEnumerable<string> AvailableNamespaces => defaultNamespaceProvider.AvailableNamespaces.Concat(new [] {
public IEnumerable<string> AvailableNamespaces => defaultNamespaceProvider.AvailableNamespaces.Concat(new[] {
StorageNamespaceType.BuiltInName,
AadNamespaceType.BuiltInName,
});

public NamespaceType? TryGetNamespace(string providerName, string aliasName, ResourceScope resourceScope, IFeatureProvider featureProvider, BicepSourceFileKind sourceFileKind)
public NamespaceType? TryGetNamespace(
string providerName,
string aliasName,
ResourceScope resourceScope,
IFeatureProvider featureProvider,
BicepSourceFileKind sourceFileKind,
string? version = null)
{
if (defaultNamespaceProvider.TryGetNamespace(providerName, aliasName, resourceScope, featureProvider, sourceFileKind) is { } namespaceType)
{
Expand Down
2 changes: 1 addition & 1 deletion src/Bicep.Core.IntegrationTests/ExtensibilityTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ public class ExtensibilityTests

private ServiceBuilder Services => new ServiceBuilder()
.WithFeatureOverrides(new(ExtensibilityEnabled: true))
.WithNamespaceProvider(new TestExtensibilityNamespaceProvider(BicepTestConstants.AzResourceTypeLoader));
.WithNamespaceProvider(new TestExtensibilityNamespaceProvider(BicepTestConstants.AzResourceTypeLoaderFactory));

[TestMethod]
public void Storage_import_bad_config_is_blocked()
Expand Down
5 changes: 3 additions & 2 deletions src/Bicep.Core.IntegrationTests/ImportTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using Bicep.Core.Features;
using Bicep.Core.Semantics;
using Bicep.Core.Semantics.Namespaces;
using Bicep.Core.Syntax;
using Bicep.Core.TypeSystem;
using Bicep.Core.UnitTests;
using Bicep.Core.UnitTests.Assertions;
Expand Down Expand Up @@ -36,9 +37,9 @@ public TestNamespaceProvider(Dictionary<string, Func<string, NamespaceType>> bui

public static bool AllowImportStatements => true;

public IEnumerable<string> AvailableNamespaces => builderDict.Keys.Concat(new [] { SystemNamespaceType.BuiltInName });
public IEnumerable<string> AvailableNamespaces => builderDict.Keys.Concat(new[] { SystemNamespaceType.BuiltInName });

public NamespaceType? TryGetNamespace(string providerName, string aliasName, ResourceScope resourceScope, IFeatureProvider features, BicepSourceFileKind sourceFileKind) => providerName switch
public NamespaceType? TryGetNamespace(string providerName, string aliasName, ResourceScope resourceScope, IFeatureProvider features, BicepSourceFileKind sourceFileKind, string? providerVersion = null) => providerName switch
{
SystemNamespaceType.BuiltInName => SystemNamespaceType.Create(aliasName, features, sourceFileKind),
{ } _ when builderDict.TryGetValue(providerName) is { } builderFunc => builderFunc(aliasName),
Expand Down
59 changes: 39 additions & 20 deletions src/Bicep.Core.IntegrationTests/OutputsTests.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

using System.Diagnostics.CodeAnalysis;
using Bicep.Core.Diagnostics;
using Bicep.Core.IntegrationTests.Extensibility;
Expand All @@ -16,19 +17,21 @@ namespace Bicep.Core.IntegrationTests
[TestClass]
public class OutputsTests
{
private ServiceBuilder ServicesWithResourceTyped => new ServiceBuilder().WithFeatureOverrides(new(TestContext, ResourceTypedParamsAndOutputsEnabled: true));
private ServiceBuilder ServicesWithResourceTyped => new ServiceBuilder()
.WithFeatureOverrides(new(TestContext, ResourceTypedParamsAndOutputsEnabled: true));

[NotNull]
public TestContext? TestContext { get; set; }

private ServiceBuilder ServicesWithExtensibility => new ServiceBuilder()
.WithFeatureOverrides(new(TestContext, ExtensibilityEnabled: true, ResourceTypedParamsAndOutputsEnabled: true))
.WithNamespaceProvider(new TestExtensibilityNamespaceProvider(BicepTestConstants.AzResourceTypeLoader));
.WithNamespaceProvider(new TestExtensibilityNamespaceProvider(BicepTestConstants.AzResourceTypeLoaderFactory));

[TestMethod]
public void Output_can_have_inferred_resource_type()
{
var result = CompilationHelper.Compile(ServicesWithResourceTyped, @"
var result = CompilationHelper.Compile(ServicesWithResourceTyped,
"""
resource resource 'Microsoft.Storage/storageAccounts@2019-06-01' = {
name: 'test'
location: 'eastus'
Expand All @@ -44,12 +47,14 @@ public void Output_can_have_inferred_resource_type()
}
}
output out resource = resource
");
""");

result.ExcludingLinterDiagnostics().Should().NotHaveAnyDiagnostics();

var model = result.Compilation.GetEntrypointSemanticModel();
var @out = model.Root.OutputDeclarations.Should().ContainSingle().Subject;
var typeInfo = model.GetTypeInfo(@out.DeclaringSyntax);

typeInfo.Should().BeOfType<ResourceType>().Which.TypeReference.FormatName().Should().BeEquivalentTo("Microsoft.Storage/storageAccounts@2019-06-01");

result.Template.Should().HaveValueAtPath("$.outputs.out", new JObject()
Expand All @@ -66,7 +71,8 @@ public void Output_can_have_inferred_resource_type()
[TestMethod]
public void Output_can_have_specified_resource_type()
{
var result = CompilationHelper.Compile(ServicesWithResourceTyped, @"
var result = CompilationHelper.Compile(ServicesWithResourceTyped,
"""
resource resource 'Microsoft.Storage/storageAccounts@2019-06-01' = {
name: 'test'
location: 'eastus'
Expand All @@ -82,12 +88,14 @@ public void Output_can_have_specified_resource_type()
}
}
output out resource 'Microsoft.Storage/storageAccounts@2019-06-01' = resource
");
""");

result.ExcludingLinterDiagnostics().Should().NotHaveAnyDiagnostics();

var model = result.Compilation.GetEntrypointSemanticModel();
var @out = model.Root.OutputDeclarations.Should().ContainSingle().Subject;
var typeInfo = model.GetTypeInfo(@out.DeclaringSyntax);

typeInfo.Should().BeOfType<ResourceType>().Which.TypeReference.FormatName().Should().BeEquivalentTo("Microsoft.Storage/storageAccounts@2019-06-01");

result.Template.Should().HaveValueAtPath("$.outputs.out", new JObject()
Expand All @@ -108,7 +116,8 @@ public void Output_can_have_specified_resource_type()
public void Output_can_have_object_type(bool enableResourceTypeParameters)
{
var services = enableResourceTypeParameters ? ServicesWithResourceTyped : new ServiceBuilder();
var result = CompilationHelper.Compile(services, @"
var result = CompilationHelper.Compile(services,
"""
resource resource 'Microsoft.Storage/storageAccounts@2019-06-01' = {
name: 'test'
location: 'eastus'
Expand All @@ -124,7 +133,8 @@ public void Output_can_have_object_type(bool enableResourceTypeParameters)
}
}
output out object = resource
");
""");

result.ExcludingLinterDiagnostics().Should().NotHaveAnyDiagnostics();

result.Template.Should().HaveValueAtPath("$.outputs.out", new JObject()
Expand All @@ -137,7 +147,8 @@ public void Output_can_have_object_type(bool enableResourceTypeParameters)
[TestMethod]
public void Output_can_have_decorators()
{
var result = CompilationHelper.Compile(ServicesWithResourceTyped, @"
var result = CompilationHelper.Compile(ServicesWithResourceTyped,
"""
resource resource 'Microsoft.Storage/storageAccounts@2019-06-01' = {
name: 'test'
location: 'eastus'
Expand All @@ -155,7 +166,8 @@ public void Output_can_have_decorators()
@description('cool beans')
output out resource 'Microsoft.Storage/storageAccounts@2019-06-01' = resource
");
""");

result.ExcludingLinterDiagnostics().Should().NotHaveAnyDiagnostics();

result.Template.Should().HaveValueAtPath("$.outputs.out", new JObject()
Expand All @@ -173,13 +185,15 @@ public void Output_can_have_decorators()
[TestMethod]
public void Output_can_have_warnings_for_missing_type()
{
var result = CompilationHelper.Compile(ServicesWithResourceTyped, @"
var result = CompilationHelper.Compile(ServicesWithResourceTyped,
"""
resource resource 'Some.Fake/Type@2019-06-01' = {
name: 'test'
}
output out resource 'Some.Fake/Type@2019-06-01' = resource
");
result.ExcludingLinterDiagnostics().Should().HaveDiagnostics(new []
""");

result.ExcludingLinterDiagnostics().Should().HaveDiagnostics(new[]
{
// There are two warnings because there are two places in code to correct the missing type.
("BCP081", DiagnosticLevel.Warning, "Resource type \"Some.Fake/Type@2019-06-01\" does not have types available."),
Expand All @@ -192,13 +206,15 @@ public void Output_can_have_warnings_for_missing_type_but_we_dont_duplicate_them
{
// As a special case we don't show a warning on the output when the type is inferred
// the user only has one location in code to correct.
var result = CompilationHelper.Compile(ServicesWithResourceTyped, @"
var result = CompilationHelper.Compile(ServicesWithResourceTyped,
"""
resource resource 'Some.Fake/Type@2019-06-01' = {
name: 'test'
}
output out resource = resource
");
result.ExcludingLinterDiagnostics().Should().HaveDiagnostics(new []
""");

result.ExcludingLinterDiagnostics().Should().HaveDiagnostics(new[]
{
("BCP081", DiagnosticLevel.Warning, "Resource type \"Some.Fake/Type@2019-06-01\" does not have types available."),
});
Expand All @@ -207,20 +223,23 @@ public void Output_can_have_warnings_for_missing_type_but_we_dont_duplicate_them
[TestMethod]
public void Output_cannot_use_extensibility_resource_type()
{
var result = CompilationHelper.Compile(ServicesWithExtensibility, @"
var result = CompilationHelper.Compile(ServicesWithExtensibility,
"""
import '[email protected]' with {
connectionString: 'asdf'
} as stg
resource container 'stg:container' = {
name: 'myblob'
}
output out resource = container
");
result.ExcludingLinterDiagnostics().Should().HaveDiagnostics(new []
""");

result.ExcludingLinterDiagnostics().Should().HaveDiagnostics(new[]
{
("BCP227", DiagnosticLevel.Error, "The type \"container\" cannot be used as a parameter or output type. Extensibility types are currently not supported as parameters or outputs."),
});
}
}
}
}
14 changes: 7 additions & 7 deletions src/Bicep.Core.IntegrationTests/ParametersTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ public class ParameterTests

private ServiceBuilder ServicesWithExtensibility => new ServiceBuilder()
.WithFeatureOverrides(new(TestContext, ExtensibilityEnabled: true, ResourceTypedParamsAndOutputsEnabled: true))
.WithNamespaceProvider(new TestExtensibilityNamespaceProvider(BicepTestConstants.AzResourceTypeLoader));
.WithNamespaceProvider(new TestExtensibilityNamespaceProvider(BicepTestConstants.AzResourceTypeLoaderFactory));

[TestMethod]
public void Parameter_can_have_resource_type()
Expand Down Expand Up @@ -158,7 +158,7 @@ param p resource 'Some.Fake/Type@2019-06-01'
output id string = p.id
");
result.ExcludingLinterDiagnostics().Should().HaveDiagnostics(new []
result.ExcludingLinterDiagnostics().Should().HaveDiagnostics(new[]
{
("BCP081", DiagnosticLevel.Warning, "Resource type \"Some.Fake/Type@2019-06-01\" does not have types available."),
});
Expand All @@ -175,7 +175,7 @@ public void Parameter_cannot_use_extensibility_resource_type()
param container resource 'stg:container'
output name string = container.name // silence unused params warning
");
result.ExcludingLinterDiagnostics().Should().HaveDiagnostics(new []
result.ExcludingLinterDiagnostics().Should().HaveDiagnostics(new[]
{
("BCP227", DiagnosticLevel.Error, "The type \"container\" cannot be used as a parameter or output type. Extensibility types are currently not supported as parameters or outputs."),
("BCP062", DiagnosticLevel.Error, "The referenced declaration with name \"container\" is not valid."),
Expand All @@ -193,7 +193,7 @@ param p resource 'Microsoft.Storage/storageAccounts@2019-06-01'
name: 'resource'
}");

result.ExcludingLinterDiagnostics().Should().HaveDiagnostics(new []
result.ExcludingLinterDiagnostics().Should().HaveDiagnostics(new[]
{
("BCP081", DiagnosticLevel.Warning, "Resource type \"My.Rp/myResource@2020-01-01\" does not have types available."),
("BCP229", DiagnosticLevel.Error, "The parameter \"p\" cannot be used as a resource scope or parent. Resources passed as parameters cannot be used as a scope or parent of a resource."),
Expand All @@ -216,7 +216,7 @@ param p resource 'Microsoft.Storage/storageAccounts@2019-06-01'
}
}");

result.ExcludingLinterDiagnostics().Should().HaveDiagnostics(new []
result.ExcludingLinterDiagnostics().Should().HaveDiagnostics(new[]
{
("BCP081", DiagnosticLevel.Warning, "Resource type \"Microsoft.Storage/storageAccounts/tableServices@2020-06-01\" does not have types available."),
("BCP229", DiagnosticLevel.Error, "The parameter \"p\" cannot be used as a resource scope or parent. Resources passed as parameters cannot be used as a scope or parent of a resource."),
Expand Down Expand Up @@ -278,7 +278,7 @@ param foo string
param bar string
"));

result.ExcludingLinterDiagnostics().Should().HaveDiagnostics(new [] {
result.ExcludingLinterDiagnostics().Should().HaveDiagnostics(new[] {
("BCP065", DiagnosticLevel.Error, "Function \"utcNow\" is not valid at this location. It can only be used as a parameter default value."),
("BCP065", DiagnosticLevel.Error, "Function \"newGuid\" is not valid at this location. It can only be used as a parameter default value."),
});
Expand All @@ -299,7 +299,7 @@ param foo string
param bar object
"));

result.ExcludingLinterDiagnostics().Should().HaveDiagnostics(new [] {
result.ExcludingLinterDiagnostics().Should().HaveDiagnostics(new[] {
("BCP057", DiagnosticLevel.Error, "The name \"resourceId\" does not exist in the current context."),
("BCP057", DiagnosticLevel.Error, "The name \"deployment\" does not exist in the current context."),
});
Expand Down
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using Bicep.Core.Configuration;
using Bicep.Core.FileSystem;
using Bicep.Core.Samples;
using Bicep.Core.Semantics;
using Bicep.Core.Text;
using Bicep.Core.UnitTests;
using Bicep.Core.UnitTests.Assertions;
using Bicep.Core.UnitTests.Utils;
using Bicep.Core.Workspaces;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
using System.Diagnostics.CodeAnalysis;
Expand Down
Loading

0 comments on commit b939e3d

Please sign in to comment.