Skip to content

Commit

Permalink
Provider restore for resource type providers (#11458)
Browse files Browse the repository at this point in the history 
## Description

This change enables the restoration of provider artifacts from the bicep
registry. In its current (temporary) form the provider artifact is
disguised as a module given that the handling of OCI Artifacts by the
ModuleRegistry is not decoupled from the handling of modules.
Refactoring (in separate PRs) is needed to decouple the concerns and
have a simpler handling of the provider artifacts.


###### Microsoft Reviewers:
codeflow:open?pullrequest=#11458

---------

Co-authored-by: Ariel Silverman <[email protected]>
  • Loading branch information
@asilverman
asilverman and Ariel Silverman authored Aug 16, 2023
1 parent 6c3a71d commit a67c321
Show file tree
Hide file tree
Showing 60 changed files with 1,046 additions and 644 deletions.
341 changes: 220 additions & 121 deletions src/Bicep.Cli.IntegrationTests/BuildCommandTests.cs
View file

Large diffs are not rendered by default.

23 changes: 12 additions & 11 deletions src/Bicep.Cli.IntegrationTests/RestoreCommandTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,14 +157,13 @@ public async Task Restore_ArtifactWithoutArtifactType_ShouldSucceed()

using (var compiledStream = new BufferedMemoryStream())
{
OciArtifactModuleReference.TryParse(null, $"{registry}/{repository}:v1", configuration, new Uri("file:///main.bicep"), out var moduleReference, out _).Should().BeTrue();

OciModuleReference.TryParse(null, $"{registry}/{repository}:v1", configuration, new Uri("file:///main.bicep"), out var artifactReference, out _).Should().BeTrue();
compiledStream.Write(TemplateEmitter.UTF8EncodingWithoutBom.GetBytes(dataSet.Compiled!));
compiledStream.Position = 0;

await containerRegistryManager.PushArtifactAsync(
configuration: configuration,
moduleReference: moduleReference!,
artifactReference: artifactReference!,
// intentionally setting artifactType to null to simulate a publish done by an older version of Bicep
artifactType: null,
config: new StreamDescriptor(Stream.Null, BicepMediaTypes.BicepModuleConfigV1),
Expand Down Expand Up @@ -219,14 +218,14 @@ public async Task Restore_With_Force_Should_Overwrite_Existing_Cache()
Directory.CreateDirectory(tempDirectory);

var publishedBicepFilePath = Path.Combine(tempDirectory, "module.bicep");
File.WriteAllText(publishedBicepFilePath,@"
File.WriteAllText(publishedBicepFilePath, @"
param p1 string
output o1 string = p1");

var (publishOutput, publishError, publishResult) = await Bicep(settings, "publish", publishedBicepFilePath, "--target", $"br:{registry}/{repository}:v1");
var (publishOutput, publishError, exitCode) = await Bicep(settings, "publish", publishedBicepFilePath, "--target", $"br:{registry}/{repository}:v1");
using (new AssertionScope())
{
publishResult.Should().Be(0);
exitCode.Should().Be(0);
publishOutput.Should().BeEmpty();
publishError.Should().BeEmpty();
}
Expand Down Expand Up @@ -273,10 +272,10 @@ param p1 string
param p2 string
output o1 string = '${p1}${p2}'");

(publishOutput, publishError, publishResult) = await Bicep(settings, "publish", publishedBicepFilePath, "--target", $"br:{registry}/{repository}:v1", "--force");
(publishOutput, publishError, exitCode) = await Bicep(settings, "publish", publishedBicepFilePath, "--target", $"br:{registry}/{repository}:v1", "--force");
using (new AssertionScope())
{
publishResult.Should().Be(0);
exitCode.Should().Be(0);
publishOutput.Should().BeEmpty();
publishError.Should().BeEmpty();
}
Expand Down Expand Up @@ -386,13 +385,15 @@ public async Task Restore_NonExistentModules_ShouldFail(DataSet dataSet)

var settings = new InvocationSettings(new(TestContext, RegistryEnabled: dataSet.HasExternalModules), clientFactory, templateSpecRepositoryFactory);
TestContext.WriteLine($"Cache root = {settings.FeatureOverrides.CacheRootDirectory}");
var (output, error, result) = await Bicep(settings, "restore", bicepFilePath);
var (output, error, exitCode) = await Bicep(settings, "restore", bicepFilePath);

using (new AssertionScope())
{
result.Should().Be(1);
exitCode.Should().Be(1);
output.Should().BeEmpty();
error.Should().ContainAll(": Error BCP192: Unable to restore the module with reference ", "The module does not exist in the registry.");
error.Should().ContainAll(": Error BCP192: Unable to restore the module with reference ", "The artifact does not exist in the registry.");


}
}

Expand Down
2 changes: 0 additions & 2 deletions src/Bicep.Cli/Commands/PublishCommand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,8 @@
using Bicep.Core.Exceptions;
using Bicep.Core.FileSystem;
using Bicep.Core.Modules;
using Bicep.Core.Parsing;
using Bicep.Core.Registry;
using System;
using System.Data.Common;
using System.IO;
using System.IO.Abstractions;
using System.Threading.Tasks;
Expand Down
18 changes: 12 additions & 6 deletions src/Bicep.Cli/Services/CompilationService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using Bicep.Core.Configuration;
using Bicep.Core.Diagnostics;
using Bicep.Core.Extensions;
using Bicep.Core.Features;
using Bicep.Core.FileSystem;
using Bicep.Core.Navigation;
using Bicep.Core.Registry;
Expand All @@ -29,6 +30,7 @@ public class CompilationService
private readonly IDiagnosticLogger diagnosticLogger;
private readonly IModuleDispatcher moduleDispatcher;
private readonly IConfigurationManager configurationManager;
private readonly IFeatureProviderFactory featureProviderFactory;
private readonly Workspace workspace;

public CompilationService(
Expand All @@ -37,7 +39,8 @@ public CompilationService(
BicepparamDecompiler paramDecompiler,
IDiagnosticLogger diagnosticLogger,
IModuleDispatcher moduleDispatcher,
IConfigurationManager configurationManager)
IConfigurationManager configurationManager,
IFeatureProviderFactory featureProviderFactory)
{
this.bicepCompiler = bicepCompiler;
this.decompiler = decompiler;
Expand All @@ -46,6 +49,7 @@ public CompilationService(
this.moduleDispatcher = moduleDispatcher;
this.configurationManager = configurationManager;
this.workspace = new Workspace();
this.featureProviderFactory = featureProviderFactory;
}

public async Task RestoreAsync(string inputPath, bool forceModulesRestore)
Expand All @@ -65,7 +69,7 @@ public async Task RestoreAsync(string inputPath, bool forceModulesRestore)
await moduleDispatcher.RestoreModules(modulesToRestoreReferences, forceModulesRestore);

// update the errors based on restore status
var sourceFileGrouping = SourceFileGroupingBuilder.Rebuild(this.moduleDispatcher, this.workspace, compilation.SourceFileGrouping);
var sourceFileGrouping = SourceFileGroupingBuilder.Rebuild(featureProviderFactory, this.moduleDispatcher, this.workspace, compilation.SourceFileGrouping);

LogDiagnostics(GetModuleRestoreDiagnosticsByBicepFile(sourceFileGrouping, originalModulesToRestore, forceModulesRestore));
}
Expand Down Expand Up @@ -131,16 +135,18 @@ public DecompileResult DecompileParams(string inputPath, string outputPath, stri
return decompilation;
}

private static ImmutableDictionary<BicepSourceFile, ImmutableArray<IDiagnostic>> GetModuleRestoreDiagnosticsByBicepFile(SourceFileGrouping sourceFileGrouping, ImmutableHashSet<ModuleSourceResolutionInfo> originalModulesToRestore, bool forceModulesRestore)
private static ImmutableDictionary<BicepSourceFile, ImmutableArray<IDiagnostic>> GetModuleRestoreDiagnosticsByBicepFile(SourceFileGrouping sourceFileGrouping, ImmutableHashSet<ArtifactResolutionInfo> originalModulesToRestore, bool forceModulesRestore)
{
static IDiagnostic? DiagnosticForModule(SourceFileGrouping grouping, IForeignTemplateReference module)
static IDiagnostic? DiagnosticForModule(SourceFileGrouping grouping, IForeignArtifactReference module)
=> grouping.TryGetErrorDiagnostic(module) is { } errorBuilder ? errorBuilder(DiagnosticBuilder.ForPosition(module.ReferenceSourceSyntax)) : null;

static IEnumerable<(BicepFile, IDiagnostic)> GetDiagnosticsForModulesToRestore(SourceFileGrouping grouping, ImmutableHashSet<ModuleSourceResolutionInfo> originalModulesToRestore)
static IEnumerable<(BicepFile, IDiagnostic)> GetDiagnosticsForModulesToRestore(SourceFileGrouping grouping, ImmutableHashSet<ArtifactResolutionInfo> originalArtifactsToRestore)
{
var originalModulesToRestore = originalArtifactsToRestore.OfType<ArtifactResolutionInfo>();
foreach (var (module, sourceFile) in originalModulesToRestore)
{
if (sourceFile is BicepFile bicepFile && DiagnosticForModule(grouping, module) is { } diagnostic)
if (sourceFile is BicepFile bicepFile &&
DiagnosticForModule(grouping, module) is { } diagnostic)
{
yield return (bicepFile, diagnostic);
}
Expand Down
17 changes: 6 additions & 11 deletions src/Bicep.Core.IntegrationTests/Emit/TemplateEmitterTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,22 +13,17 @@
using Bicep.Core.Parsing;
using Bicep.Core.Registry;
using Bicep.Core.Samples;
using Bicep.Core.Semantics;
using Bicep.Core.UnitTests;
using Bicep.Core.UnitTests.Assertions;
using Bicep.Core.UnitTests.Features;
using Bicep.Core.UnitTests.Mock;
using Bicep.Core.UnitTests.Utils;
using Bicep.Core.Workspaces;
using FluentAssertions;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using FluentAssertions.Execution;
using Bicep.Core.UnitTests.Baselines;
using System;
using System.Reflection;
using System.Text.RegularExpressions;


namespace Bicep.Core.IntegrationTests.Emit
{
Expand All @@ -53,10 +48,10 @@ private async Task<SourceFileGrouping> GetSourceFileGrouping(DataSet dataSet)
var configManager = BicepTestConstants.CreateFilesystemConfigurationManager();
var dispatcher = new ModuleDispatcher(new DefaultModuleRegistryProvider(BicepTestConstants.EmptyServiceProvider, BicepTestConstants.FileResolver, clientFactory, templateSpecRepositoryFactory, BicepTestConstants.CreateFeatureProviderFactory(new(TestContext, RegistryEnabled: dataSet.HasExternalModules), configManager), configManager), configManager);
Workspace workspace = new();
var sourceFileGrouping = SourceFileGroupingBuilder.Build(BicepTestConstants.FileResolver, dispatcher, workspace, PathHelper.FilePathToFileUrl(bicepFilePath));
var sourceFileGrouping = SourceFileGroupingBuilder.Build(BicepTestConstants.FileResolver, dispatcher, workspace, PathHelper.FilePathToFileUrl(bicepFilePath), BicepTestConstants.FeatureProviderFactory);
if (await dispatcher.RestoreModules(dispatcher.GetValidModuleReferences(sourceFileGrouping.GetModulesToRestore())))
{
sourceFileGrouping = SourceFileGroupingBuilder.Rebuild(dispatcher, workspace, sourceFileGrouping);
sourceFileGrouping = SourceFileGroupingBuilder.Rebuild(BicepTestConstants.FeatureProviderFactory, dispatcher, workspace, sourceFileGrouping);
}

return sourceFileGrouping;
Expand Down Expand Up @@ -222,7 +217,7 @@ public void InvalidBicep_TemplateEmiterShouldNotProduceAnyTemplate(DataSet dataS

// emitting the template should fail
var dispatcher = new ModuleDispatcher(BicepTestConstants.RegistryProvider, IConfigurationManager.WithStaticConfiguration(BicepTestConstants.BuiltInConfigurationWithAllAnalyzersDisabled));
var result = this.EmitTemplate(SourceFileGroupingBuilder.Build(BicepTestConstants.FileResolver, dispatcher, new Workspace(), PathHelper.FilePathToFileUrl(bicepFilePath)), new(), filePath);
var result = this.EmitTemplate(SourceFileGroupingBuilder.Build(BicepTestConstants.FileResolver, dispatcher, new Workspace(), PathHelper.FilePathToFileUrl(bicepFilePath), BicepTestConstants.FeatureProviderFactory), new(), filePath);
result.Diagnostics.Should().NotBeEmpty();
result.Status.Should().Be(EmitStatus.Failed);
}
Expand All @@ -235,7 +230,7 @@ public void Valid_bicepparam_TemplateEmiter_should_produce_expected_template(Bas
var data = baselineData.GetData(TestContext);
data.Compiled.Should().NotBeNull();

var sourceFileGrouping = SourceFileGroupingBuilder.Build(BicepTestConstants.FileResolver, BicepTestConstants.ModuleDispatcher, new Workspace(), PathHelper.FilePathToFileUrl(data.Parameters.OutputFilePath));
var sourceFileGrouping = SourceFileGroupingBuilder.Build(BicepTestConstants.FileResolver, BicepTestConstants.ModuleDispatcher, new Workspace(), PathHelper.FilePathToFileUrl(data.Parameters.OutputFilePath), BicepTestConstants.FeatureProviderFactory);
var result = this.EmitParam(sourceFileGrouping, data.Compiled!.OutputFilePath);

result.Diagnostics.Should().NotHaveErrors();
Expand All @@ -251,7 +246,7 @@ public void Invalid_bicepparam_TemplateEmiter_should_not_produce_a_template(Base
{
var data = baselineData.GetData(TestContext);

var sourceFileGrouping = SourceFileGroupingBuilder.Build(BicepTestConstants.FileResolver, BicepTestConstants.ModuleDispatcher, new Workspace(), PathHelper.FilePathToFileUrl(data.Parameters.OutputFilePath));
var sourceFileGrouping = SourceFileGroupingBuilder.Build(BicepTestConstants.FileResolver, BicepTestConstants.ModuleDispatcher, new Workspace(), PathHelper.FilePathToFileUrl(data.Parameters.OutputFilePath), BicepTestConstants.FeatureProviderFactory);
var result = this.EmitParam(sourceFileGrouping, Path.ChangeExtension(data.Parameters.OutputFilePath, ".json"));

result.Diagnostics.Should().NotBeEmpty();
Expand Down
2 changes: 1 addition & 1 deletion src/Bicep.Core.IntegrationTests/ExamplesTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ private void RunExampleTest(EmbeddedFile embeddedBicep, FeatureProviderOverrides

var configManager = IConfigurationManager.WithStaticConfiguration(BicepTestConstants.BuiltInConfigurationWithAllAnalyzersDisabled);
var dispatcher = new ModuleDispatcher(BicepTestConstants.RegistryProvider, configManager);
var sourceFileGrouping = SourceFileGroupingBuilder.Build(BicepTestConstants.FileResolver, dispatcher, new Workspace(), PathHelper.FilePathToFileUrl(bicepFile.OutputFilePath));
var sourceFileGrouping = SourceFileGroupingBuilder.Build(BicepTestConstants.FileResolver, dispatcher, new Workspace(), PathHelper.FilePathToFileUrl(bicepFile.OutputFilePath), BicepTestConstants.FeatureProviderFactory);
var compilation = Services.WithFeatureOverrides(features).Build().BuildCompilation(sourceFileGrouping);
var emitter = new TemplateEmitter(compilation.GetEntrypointSemanticModel());

Expand Down
2 changes: 1 addition & 1 deletion src/Bicep.Core.IntegrationTests/ModuleTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ public void SourceFileGroupingBuilder_build_should_throw_diagnostic_exception_if
var mockDispatcher = Repository.Create<IModuleDispatcher>();
SetupFileReaderMock(mockFileResolver, fileUri, null, x => x.ErrorOccurredReadingFile("Mock read failure!"));

Action buildAction = () => SourceFileGroupingBuilder.Build(mockFileResolver.Object, mockDispatcher.Object, new Workspace(), fileUri);
Action buildAction = () => SourceFileGroupingBuilder.Build(mockFileResolver.Object, mockDispatcher.Object, new Workspace(), fileUri, BicepTestConstants.FeatureProviderFactory);
buildAction.Should().Throw<ErrorDiagnosticException>()
.And.Diagnostic.Should().HaveCodeAndSeverity("BCP091", DiagnosticLevel.Error).And.HaveMessage("An error occurred reading file. Mock read failure!");
}
Expand Down
Loading

0 comments on commit a67c321

Please sign in to comment.