From 271fefdb2d598e0c67c707566268aa279f8597ae Mon Sep 17 00:00:00 2001 From: Alexander Sehr Date: Sun, 29 Oct 2023 16:46:30 +0100 Subject: [PATCH] feat: Ported latest CARML-utilities improvements (#586) ## Description Ported latest CARML-utilities improvements - Enabled ReadMe utility to consume 'orphaned' and 'moved' readme files and put them on top of the readme - Made the ReadMe script not delete the readme but actually just overwrite it - Removed ARM support from readme utility - Added retry logic to link checker ### Pipeline references | Pipeline | | - | | [![avm.res.kubernetes-configuration.flux-configuration](https://github.com/AlexanderSehr/bicep-registry-modules/actions/workflows/avm.res.kubernetes-configuration.flux-configuration.yml/badge.svg?branch=users%2Falsehr%2FreadMeNewFiles)](https://github.com/AlexanderSehr/bicep-registry-modules/actions/workflows/avm.res.kubernetes-configuration.flux-configuration.yml) | | [![avm.res.key-vault.vault](https://github.com/AlexanderSehr/bicep-registry-modules/actions/workflows/avm.res.key-vault.vault.yml/badge.svg?branch=users%2Falsehr%2FreadMeNewFiles)](https://github.com/AlexanderSehr/bicep-registry-modules/actions/workflows/avm.res.key-vault.vault.yml) | --------- Co-authored-by: Erika Gressi <56914614+eriqua@users.noreply.github.com> --- .../flux-configuration/README.md | 5 + .../sharedScripts/Set-ModuleReadMe.ps1 | 616 +++++++----------- avm/utilities/tools/Set-AVMModule.ps1 | 21 +- 3 files changed, 247 insertions(+), 395 deletions(-) diff --git a/avm/res/kubernetes-configuration/flux-configuration/README.md b/avm/res/kubernetes-configuration/flux-configuration/README.md index 3ae5890526..3eb0a700e9 100644 --- a/avm/res/kubernetes-configuration/flux-configuration/README.md +++ b/avm/res/kubernetes-configuration/flux-configuration/README.md @@ -1,5 +1,10 @@ # Kubernetes Configuration Flux Configurations `[Microsoft.KubernetesConfiguration/fluxConfigurations]` +> ⚠️THIS MODULE IS CURRENTLY ORPHANED.⚠️ +> +> - Only security and bug fixes are being handled by the AVM core team at present. +> - If interested in becoming the module owner of this orphaned module (must be Microsoft FTE), please look for the related "orphaned module" GitHub issue [here](https://aka.ms/AVM/OrphanedModules)! + This module deploys a Kubernetes Configuration Flux Configuration. ## Navigation diff --git a/avm/utilities/pipelines/sharedScripts/Set-ModuleReadMe.ps1 b/avm/utilities/pipelines/sharedScripts/Set-ModuleReadMe.ps1 index d4f835faa4..3f3efc1535 100644 --- a/avm/utilities/pipelines/sharedScripts/Set-ModuleReadMe.ps1 +++ b/avm/utilities/pipelines/sharedScripts/Set-ModuleReadMe.ps1 @@ -1,5 +1,50 @@ #requires -version 7.3 +#region helper functions +<# +.SYNOPSIS +Test if an URL points to a valid online endpoint + +.DESCRIPTION +Test if an URL points to a valid online endpoint + +.PARAMETER URL +Mandatory. The URL to check + +.PARAMETER Retries +Optional. The amount of times to retry + +.EXAMPLE +Test-URl -URL 'www.github.com' + +Returns $true if the 'www.github.com' is valid, $false otherwise +#> +function Test-Url { + + [CmdletBinding()] + param ( + [Parameter(Mandatory = $true)] + [string] $URL, + + [Parameter(Mandatory = $false)] + [int] $Retries = 3 + ) + + $currentAttempt = 1 + + while ($currentAttempt -le $Retries) { + try { + $null = Invoke-WebRequest -Uri $URL + return $true + } catch { + $currentAttempt++ + Start-Sleep -Seconds 1 + } + } + + return $false +} + <# .SYNOPSIS Update the 'Resource Types' section of the given readme file @@ -59,27 +104,22 @@ function Set-ResourceTypesSection { $ProviderNamespace, $ResourceType = $resourceTypeObject.Type -split '/', 2 # Validate if Reference URL is working $TemplatesBaseUrl = 'https://learn.microsoft.com/en-us/azure/templates' - try { - $ResourceReferenceUrl = '{0}/{1}/{2}/{3}' -f $TemplatesBaseUrl, $ProviderNamespace, $resourceTypeObject.ApiVersion, $ResourceType - $null = Invoke-WebRequest -Uri $ResourceReferenceUrl - } - catch { + + $ResourceReferenceUrl = '{0}/{1}/{2}/{3}' -f $TemplatesBaseUrl, $ProviderNamespace, $resourceTypeObject.ApiVersion, $ResourceType + if (-not (Test-Url $ResourceReferenceUrl)) { # Validate if Reference URL is working using the latest documented API version (with no API version in the URL) - try { - $ResourceReferenceUrl = '{0}/{1}/{2}' -f $TemplatesBaseUrl, $ProviderNamespace, $ResourceType - $null = Invoke-WebRequest -Uri $ResourceReferenceUrl - } - catch { - # Check if the resource is a child resource - if ($ResourceType.Split('/').length -gt 1) { - $ResourceReferenceUrl = '{0}/{1}/{2}' -f $TemplatesBaseUrl, $ProviderNamespace, $ResourceType.Split('/')[0] - } - else { - # Use the default Templates URL (Last resort) - $ResourceReferenceUrl = '{0}' -f $TemplatesBaseUrl - } + $ResourceReferenceUrl = '{0}/{1}/{2}' -f $TemplatesBaseUrl, $ProviderNamespace, $ResourceType + } + if (-not (Test-Url $ResourceReferenceUrl)) { + # Check if the resource is a child resource + if ($ResourceType.Split('/').length -gt 1) { + $ResourceReferenceUrl = '{0}/{1}/{2}' -f $TemplatesBaseUrl, $ProviderNamespace, $ResourceType.Split('/')[0] + } else { + # Use the default Templates URL (Last resort) + $ResourceReferenceUrl = '{0}' -f $TemplatesBaseUrl } } + $SectionContent += ('| `{0}` | [{1}]({2}) |' -f $resourceTypeObject.type, $resourceTypeObject.apiVersion, $ResourceReferenceUrl) } $ProgressPreference = 'Continue' @@ -195,20 +235,16 @@ function Set-ParametersSection { $isRequired = (-not $definition['nullable']) $defaultValue = $null $rawAllowedValues = $definition['allowedValues'] - } - else { + } else { $type = $parameter.type if ($parameter.defaultValue -is [array]) { $defaultValue = '[{0}]' -f (($parameter.defaultValue | Sort-Object) -join ', ') - } - elseif ($parameter.defaultValue -is [hashtable]) { + } elseif ($parameter.defaultValue -is [hashtable]) { $defaultValue = '{object}' - } - elseif ($parameter.defaultValue -is [string] -and ($parameter.defaultValue -notmatch '\[\w+\(.*\).*\]')) { + } elseif ($parameter.defaultValue -is [string] -and ($parameter.defaultValue -notmatch '\[\w+\(.*\).*\]')) { $defaultValue = '''' + $parameter.defaultValue + '''' - } - else { + } else { $defaultValue = $parameter.defaultValue } @@ -327,8 +363,7 @@ function Set-DefinitionSection { # definition type (if any) if ($parameterValue.Keys -contains '$ref') { $definition = $TemplateFileContent.definitions[(Split-Path $parameterValue.'$ref' -Leaf)] - } - else { + } else { $definition = $null } @@ -414,8 +449,7 @@ function Set-OutputsSection { $description = $output.metadata.description.Replace("`r`n", '

').Replace("`n", '

') $SectionContent += ("| ``{0}`` | {1} | {2} |" -f $outputName, $output.type, $description) } - } - else { + } else { $SectionContent = [System.Collections.ArrayList]@( '| Output | Type |', '| :-- | :-- |' @@ -547,7 +581,10 @@ Add type comments to given bicep params string, using one required parameter 'na // Required parameters name: 'carml' // Non-required parameters - lock: 'CanNotDelete' + lock: { + kind: 'CanNotDelete' + name: 'myCustomLockName' + } ' #> function Add-BicepParameterTypeComment { @@ -587,8 +624,7 @@ function Add-BicepParameterTypeComment { if ($nextLineIndent -gt $requiredParameterIndent) { # Case Param is object/array: Search in rest of array for the next closing bracket with the same indent - and then add the search index (1) & initial index (1) count back in $requiredParameterEndIndex = ($BicepParamsArray[($requiredParameterStartIndex + 1)..($BicepParamsArray.Count)] | Select-String "^[\s]{$requiredParameterIndent}\S+" | ForEach-Object { $_.LineNumber - 1 })[0] + 1 + $requiredParameterStartIndex - } - else { + } else { # Case Param is single line bool/string/int: Add an index (1) for the 'required' comment $requiredParameterEndIndex = $requiredParameterStartIndex } @@ -623,7 +659,10 @@ Order the given JSON object alphabetically. Would result into: @{ name: 'carml' - lock: 'CanNotDelete' + lock: { + kind: 'CanNotDelete' + name: 'myCustomLockName' + } } #> function Get-OrderedParametersJSON { @@ -845,17 +884,14 @@ function ConvertTo-FormattedJSONParameterObject { if ($isLineWithObjectPropertyReference -or $isLineWithFunction -or $isLineWithParameterOrVariableReferenceValue) { $line = '{0}: "<{1}>"' -f ($line -split ':')[0], ([regex]::Match(($line -split ':')[0], '"(.+)"')).Captures.Groups[1].Value - } - elseif ($isLineWithObjectReferenceKeyAndEmptyObjectValue) { + } elseif ($isLineWithObjectReferenceKeyAndEmptyObjectValue) { $line = '"<{0}>": {1}' -f (($line -split ':')[0] -split '\.')[-1].TrimEnd('}"'), $lineValue } - } - else { + } else { if ($line -notlike '*"*"*' -and $line -like '*.*') { # In case of a array value like '[ \n -> resourceGroupResources.outputs.managedIdentityPrincipalId <- \n ]' we'll only show """ $line = '"<{0}>"' -f $line.Split('.')[-1].Trim() - } - elseif ($line -match '^\s*[a-zA-Z]+\s*$') { + } elseif ($line -match '^\s*[a-zA-Z]+\s*$') { # If there is simply only a value such as a variable reference, we'll wrap it as a string to replace. For example a reference of a variable `addressPrefix` will be replaced with `""` $line = '"<{0}>"' -f $line.Trim() } @@ -879,8 +915,7 @@ function ConvertTo-FormattedJSONParameterObject { # [2.7] Format the final JSON string to an object to enable processing try { $paramInJsonFormatObject = $paramInJSONFormatArray | Out-String | ConvertFrom-Json -AsHashtable -Depth 99 -ErrorAction 'Stop' - } - catch { + } catch { throw ('Failed to process file [{0}]. Please check if it properly formatted. Original error message: [{1}]' -f $CurrentFilePath, $_.Exception.Message) } # [3/4] Inject top-level 'value`' properties @@ -917,7 +952,10 @@ Convert the given JSONParameters object with one required parameter to a formatt // Required parameters name: 'carml' // Non-required parameters - lock: 'CanNotDelete' + lock: { + kind: 'CanNotDelete' + name: 'myCustomLockName' + } ' #> function ConvertTo-FormattedBicep { @@ -938,8 +976,7 @@ function ConvertTo-FormattedBicep { $keysOnLevel = $JSONParameters[$parameterName].Keys if ($keysOnLevel.count -eq 1 -and $keysOnLevel -eq 'value') { $JSONParametersWithoutValue[$parameterName] = $JSONParameters[$parameterName].value - } - else { + } else { $JSONParametersWithoutValue[$parameterName] = $JSONParameters[$parameterName] } } @@ -947,8 +984,7 @@ function ConvertTo-FormattedBicep { # [1/5] Order parameters recursively if ($JSONParametersWithoutValue.psbase.Keys.Count -gt 0) { $orderedJSONParameters = Get-OrderedParametersJSON -ParametersJSON ($JSONParametersWithoutValue | ConvertTo-Json -Depth 99) -RequiredParametersList $RequiredParametersList - } - else { + } else { $orderedJSONParameters = @{} } # [2/5] Remove any JSON specific formatting @@ -967,13 +1003,11 @@ function ConvertTo-FormattedBicep { if ($_ -match ".+: '(\w+)\.getSecret\(\\'([0-9a-zA-Z-<>]+)\\'\)'") { # e.g. change [pfxCertificate: 'kv1.getSecret(\'\')'] to [pfxCertificate: kv1.getSecret('')] "{0}: {1}.getSecret('{2}')" -f ($_ -split ':')[0], $matches[1], $matches[2] - } - else { + } else { $_ } } - } - else { + } else { $bicepParamsArray = @() } @@ -1074,14 +1108,10 @@ function Set-UsageExamplesSection { if ($specialConversionHash.ContainsKey($moduleName)) { # Convert moduleName using specialConversionHash $moduleNameCamelCase = $specialConversionHash[$moduleName] - $moduleNamePascalCase = $moduleNameCamelCase.Replace($moduleNameCamelCase[0], $moduleNameCamelCase[0].ToString().ToUpper()) - } - else { + } else { # Convert moduleName from kebab-case to camelCase $First, $Rest = $moduleName -Split '-', 2 $moduleNameCamelCase = $First.Tolower() + (Get-Culture).TextInfo.ToTitleCase($Rest) -Replace '-' - # Convert moduleName from kebab-case to PascalCase - $moduleNamePascalCase = (Get-Culture).TextInfo.ToTitleCase($moduleName) -Replace '-' } $testFilePaths = Get-ModuleTestFileList -ModulePath $moduleRoot | ForEach-Object { Join-Path $moduleRoot $_ } @@ -1106,12 +1136,10 @@ function Set-UsageExamplesSection { # Format example header if ($compiledTestFileContent.metadata.Keys -contains 'name') { $exampleTitle = $compiledTestFileContent.metadata.name - } - else { - if ((Split-Path (Split-Path $testFilePath -Parent) -Leaf) -ne 'tests') { + } else { + if ((Split-Path (Split-Path $testFilePath -Parent) -Leaf) -ne '.test') { $exampleTitle = Split-Path (Split-Path $testFilePath -Parent) -Leaf - } - else { + } else { $exampleTitle = ((Split-Path $testFilePath -LeafBase) -replace '\.', ' ') -replace ' parameters', '' } $textInfo = (Get-Culture -Name 'en-US').TextInfo @@ -1136,350 +1164,141 @@ function Set-UsageExamplesSection { ) } - ## ----------------------------------- ## - ## Handle by type (Bicep vs. JSON) ## - ## ----------------------------------- ## - if ((Split-Path $testFilePath -Extension) -eq '.bicep') { - - # ------------------------- # - # Prepare Bicep to JSON # - # ------------------------- # - - # [1/6] Search for the relevant parameter start & end index - $bicepTestStartIndex = ($rawContentArray | Select-String ("^module testDeployment '..\/.*main.bicep' = ") | ForEach-Object { $_.LineNumber - 1 })[0] - - $bicepTestEndIndex = $bicepTestStartIndex - do { - $bicepTestEndIndex++ - } while ($rawContentArray[$bicepTestEndIndex] -notin @('}', '}]')) - - $rawBicepExample = $rawContentArray[$bicepTestStartIndex..$bicepTestEndIndex] + # ------------------------- # + # Prepare Bicep to JSON # + # ------------------------- # - if ($rawBicepExample[-1] -eq '}]') { - $rawBicepExample[-1] = '}' - } - - # [2/6] Replace placeholders - $serviceShort = ([regex]::Match($rawContent, "(?m)^param serviceShort string = '(.+)'\s*$")).Captures.Groups[1].Value - - $rawBicepExampleString = ($rawBicepExample | Out-String) - $rawBicepExampleString = $rawBicepExampleString -replace '\$\{serviceShort\}', $serviceShort - $rawBicepExampleString = $rawBicepExampleString -replace '\$\{namePrefix\}[-|\.|_]?', '' # Replacing with empty to not expose prefix and avoid potential deployment conflicts - $rawBicepExampleString = $rawBicepExampleString -replace '(?m):\s*location\s*$', ': ''''' - $rawBicepExampleString = $rawBicepExampleString -replace '-\$\{iteration\}', '' - - # [3/6] Format header, remove scope property & any empty line - $rawBicepExample = $rawBicepExampleString -split '\n' - $rawBicepExample[0] = "module $moduleNameCamelCase 'br/public:$($brLink):1.0.0' = {" - $rawBicepExample = $rawBicepExample | Where-Object { $_ -notmatch 'scope: *' } | Where-Object { -not [String]::IsNullOrEmpty($_) } - # [4/6] Extract param block - $rawBicepExampleArray = $rawBicepExample -split '\n' - $moduleDeploymentPropertyIndent = ([regex]::Match($rawBicepExampleArray[1], '^(\s+).*')).Captures.Groups[1].Value.Length - $paramsStartIndex = ($rawBicepExampleArray | Select-String ("^[\s]{$moduleDeploymentPropertyIndent}params:[\s]*\{") | ForEach-Object { $_.LineNumber - 1 })[0] + 1 - if ($rawBicepExampleArray[$paramsStartIndex].Trim() -ne '}') { - # Handle case where param block is empty - $paramsEndIndex = ($rawBicepExampleArray[($paramsStartIndex + 1)..($rawBicepExampleArray.Count)] | Select-String "^[\s]{$moduleDeploymentPropertyIndent}\}" | ForEach-Object { $_.LineNumber - 1 })[0] + $paramsStartIndex - $paramBlock = ($rawBicepExampleArray[$paramsStartIndex..$paramsEndIndex] | Out-String).TrimEnd() - } - else { - $paramBlock = '' - $paramsEndIndex = $paramsStartIndex - } + # [1/6] Search for the relevant parameter start & end index + $bicepTestStartIndex = ($rawContentArray | Select-String ("^module testDeployment '..\/.*main.bicep' = ") | ForEach-Object { $_.LineNumber - 1 })[0] - # [5/6] Convert Bicep parameter block to JSON parameter block to enable processing - $conversionInputObject = @{ - BicepParamBlock = $paramBlock - CurrentFilePath = $testFilePath - } - $paramsInJSONFormat = ConvertTo-FormattedJSONParameterObject @conversionInputObject + $bicepTestEndIndex = $bicepTestStartIndex + do { + $bicepTestEndIndex++ + } while ($rawContentArray[$bicepTestEndIndex] -notin @('}', '}]')) - # [6/6] Convert JSON parameters back to Bicep and order & format them - $conversionInputObject = @{ - JSONParameters = $paramsInJSONFormat - RequiredParametersList = $RequiredParametersList - } - $bicepExample = ConvertTo-FormattedBicep @conversionInputObject + $rawBicepExample = $rawContentArray[$bicepTestStartIndex..$bicepTestEndIndex] - # --------------------- # - # Add Bicep example # - # --------------------- # - if ($addBicep) { + if ($rawBicepExample[-1] -eq '}]') { + $rawBicepExample[-1] = '}' + } - if ([String]::IsNullOrEmpty($paramBlock)) { - # Handle case where param block is empty - $formattedBicepExample = $rawBicepExample[0..($paramsStartIndex - 1)] + $rawBicepExample[($paramsEndIndex)..($rawBicepExample.Count)] - } - else { - $formattedBicepExample = $rawBicepExample[0..($paramsStartIndex - 1)] + ($bicepExample -split '\n') + $rawBicepExample[($paramsEndIndex + 1)..($rawBicepExample.Count)] - } + # [2/6] Replace placeholders + $serviceShort = ([regex]::Match($rawContent, "(?m)^param serviceShort string = '(.+)'\s*$")).Captures.Groups[1].Value + + $rawBicepExampleString = ($rawBicepExample | Out-String) + $rawBicepExampleString = $rawBicepExampleString -replace '\$\{serviceShort\}', $serviceShort + $rawBicepExampleString = $rawBicepExampleString -replace '\$\{namePrefix\}[-|\.|_]?', '' # Replacing with empty to not expose prefix and avoid potential deployment conflicts + $rawBicepExampleString = $rawBicepExampleString -replace '(?m):\s*location\s*$', ': ''''' + $rawBicepExampleString = $rawBicepExampleString -replace '-\$\{iteration\}', '' + + # [3/6] Format header, remove scope property & any empty line + $rawBicepExample = $rawBicepExampleString -split '\n' + $rawBicepExample[0] = "module $moduleNameCamelCase 'br/public:$($brLink):1.0.0' = {" + $rawBicepExample = $rawBicepExample | Where-Object { $_ -notmatch 'scope: *' } | Where-Object { -not [String]::IsNullOrEmpty($_) } + # [4/6] Extract param block + $rawBicepExampleArray = $rawBicepExample -split '\n' + $moduleDeploymentPropertyIndent = ([regex]::Match($rawBicepExampleArray[1], '^(\s+).*')).Captures.Groups[1].Value.Length + $paramsStartIndex = ($rawBicepExampleArray | Select-String ("^[\s]{$moduleDeploymentPropertyIndent}params:[\s]*\{") | ForEach-Object { $_.LineNumber - 1 })[0] + 1 + if ($rawBicepExampleArray[$paramsStartIndex].Trim() -ne '}') { + # Handle case where param block is empty + $paramsEndIndex = ($rawBicepExampleArray[($paramsStartIndex + 1)..($rawBicepExampleArray.Count)] | Select-String "^[\s]{$moduleDeploymentPropertyIndent}\}" | ForEach-Object { $_.LineNumber - 1 })[0] + $paramsStartIndex + $paramBlock = ($rawBicepExampleArray[$paramsStartIndex..$paramsEndIndex] | Out-String).TrimEnd() + } else { + $paramBlock = '' + $paramsEndIndex = $paramsStartIndex + } - # Remove any dependsOn as it it test specific - if ($detected = ($formattedBicepExample | Select-String "^\s{$moduleDeploymentPropertyIndent}dependsOn:\s*\[\s*$" | ForEach-Object { $_.LineNumber - 1 })) { - $dependsOnStartIndex = $detected[0] + # [5/6] Convert Bicep parameter block to JSON parameter block to enable processing + $conversionInputObject = @{ + BicepParamBlock = $paramBlock + CurrentFilePath = $testFilePath + } + $paramsInJSONFormat = ConvertTo-FormattedJSONParameterObject @conversionInputObject - # Find out where the 'dependsOn' ends - $dependsOnEndIndex = $dependsOnStartIndex - do { - $dependsOnEndIndex++ - } while ($formattedBicepExample[$dependsOnEndIndex] -notmatch '^\s*\]\s*$') + # [6/6] Convert JSON parameters back to Bicep and order & format them + $conversionInputObject = @{ + JSONParameters = $paramsInJSONFormat + RequiredParametersList = $RequiredParametersList + } + $bicepExample = ConvertTo-FormattedBicep @conversionInputObject - # Cut the 'dependsOn' block out - $formattedBicepExample = $formattedBicepExample[0..($dependsOnStartIndex - 1)] + $formattedBicepExample[($dependsOnEndIndex + 1)..($formattedBicepExample.Count)] - } + # --------------------- # + # Add Bicep example # + # --------------------- # + if ($addBicep) { - # Build result - $testFilesContent += @( - '', - '

' - '' - 'via Bicep module' - '' - '```bicep', - ($formattedBicepExample | ForEach-Object { "$_" }).TrimEnd(), - '```', - '', - '
', - '

' - ) + if ([String]::IsNullOrEmpty($paramBlock)) { + # Handle case where param block is empty + $formattedBicepExample = $rawBicepExample[0..($paramsStartIndex - 1)] + $rawBicepExample[($paramsEndIndex)..($rawBicepExample.Count)] + } else { + $formattedBicepExample = $rawBicepExample[0..($paramsStartIndex - 1)] + ($bicepExample -split '\n') + $rawBicepExample[($paramsEndIndex + 1)..($rawBicepExample.Count)] } - # -------------------- # - # Add JSON example # - # -------------------- # - if ($addJson) { + # Remove any dependsOn as it it test specific + if ($detected = ($formattedBicepExample | Select-String "^\s{$moduleDeploymentPropertyIndent}dependsOn:\s*\[\s*$" | ForEach-Object { $_.LineNumber - 1 })) { + $dependsOnStartIndex = $detected[0] - # [1/2] Get all parameters from the parameter object and order them recursively - $orderingInputObject = @{ - ParametersJSON = $paramsInJSONFormat | ConvertTo-Json -Depth 99 - RequiredParametersList = $RequiredParametersList - } - $orderedJSONExample = Build-OrderedJSONObject @orderingInputObject + # Find out where the 'dependsOn' ends + $dependsOnEndIndex = $dependsOnStartIndex + do { + $dependsOnEndIndex++ + } while ($formattedBicepExample[$dependsOnEndIndex] -notmatch '^\s*\]\s*$') - # [2/2] Create the final content block - $testFilesContent += @( - '', - '

' - '' - 'via JSON Parameter file' - '' - '```json', - $orderedJSONExample.Trim() - '```', - '', - '
', - '

' - ) + # Cut the 'dependsOn' block out + $formattedBicepExample = $formattedBicepExample[0..($dependsOnStartIndex - 1)] + $formattedBicepExample[($dependsOnEndIndex + 1)..($formattedBicepExample.Count)] } - } - else { - # ------------------------- # - # Prepare JSON to Bicep # - # ------------------------- # - - $rawContentHashtable = $rawContent | ConvertFrom-Json -Depth 99 -AsHashtable -NoEnumerate - - # First we need to check if we're dealing with classic JSON-Parameter file, or a deployment test file (which contains resource deployments & parameters) - $isParameterFile = $rawContentHashtable.'$schema' -like '*deploymentParameters*' - if (-not $isParameterFile) { - # Case 1: Uses deployment test file (instead of parameter file). - # [1/4] Need to extract parameters. The target is to get an object which 1:1 represents a classic JSON-Parameter file (aside from KeyVault references) - $testResource = $rawContentHashtable.resources | Where-Object { $_.name -like '*-test-*' } - - # [2/4] Build the full ARM-JSON parameter file - $jsonParameterContent = [ordered]@{ - '$schema' = 'https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#' - contentVersion = '1.0.0.0' - parameters = $testResource.properties.parameters - } - $jsonParameterContent = ($jsonParameterContent | ConvertTo-Json -Depth 99).TrimEnd() - - # [3/4] Remove 'externalResourceReferences' that are generated for Bicep's 'existing' resource references. Removing them will make the file more readable - $jsonParameterContentArray = $jsonParameterContent -split '\n' - foreach ($row in ($jsonParameterContentArray | Where-Object { $_ -like '*reference(extensionResourceId*' })) { - if ($row -match '\[.*reference\(extensionResourceId.+\.([a-zA-Z]+)\..*\].*"') { - # e.g. "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', format('{0}-diagnosticDependencies', uniqueString(deployment().name, parameters('location')))), '2020-10-01').outputs.logAnalyticsWorkspaceResourceId.value]" - # e.g. "[format('{0}', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', format('{0}-paramNested', uniqueString(deployment().name, parameters('location')))), '2020-10-01').outputs.managedIdentityResourceId.value)]": {} - $expectedValue = $matches[1] - } - elseif ($row -match '\[.*reference\(extensionResourceId.+\.([a-zA-Z]+).*\].*"') { - # e.g. "[reference(extensionResourceId(managementGroup().id, 'Microsoft.Authorization/policySetDefinitions', format('dep-#_namePrefix_#-polSet-{0}', parameters('serviceShort'))), '2021-06-01').policyDefinitions[0].policyDefinitionReferenceId]" - $expectedValue = $matches[1] - } - else { - throw "Unhandled case [$row] in file [$testFilePath]" - } - - $toReplaceValue = ([regex]::Match($row, '"(\[.+)"')).Captures.Groups[1].Value - - $jsonParameterContent = $jsonParameterContent.Replace($toReplaceValue, ('<{0}>' -f $expectedValue)) - } - # [4/4] Removing template specific functions - $jsonParameterContentArray = $jsonParameterContent -split '\n' - for ($index = 0; $index -lt $jsonParameterContentArray.Count; $index++) { - if ($jsonParameterContentArray[$index] -match '(\s*"value"): "\[.+\]"') { - # e.g. - # "policyAssignmentId": { - # "value": "[extensionResourceId(managementGroup().id, 'Microsoft.Authorization/policyAssignments', format('dep-#_namePrefix_#-psa-{0}', parameters('serviceShort')))]" - $prefix = $matches[1] - - $headerIndex = $index - while (($jsonParameterContentArray[$headerIndex] -notmatch '.+": (\{|\[)+' -or $jsonParameterContentArray[$headerIndex] -like '*"value"*') -and $headerIndex -gt -1) { - $headerIndex-- - } - - $value = (($jsonParameterContentArray[$headerIndex] -split ':')[0] -replace '"').Trim() - $jsonParameterContentArray[$index] = ('{0}: "<{1}>"{2}' -f $prefix, $value, ($jsonParameterContentArray[$index].Trim() -like '*,' ? ',' : '')) - } - elseif ($jsonParameterContentArray[$index] -match '(\s*)"([\w]+)": "\[.+\]"') { - # e.g. "name": "[format('{0}01', parameters('serviceShort'))]" - $jsonParameterContentArray[$index] = ('{0}"{1}": "<{1}>"{2}' -f $matches[1], $matches[2], ($jsonParameterContentArray[$index].Trim() -like '*,' ? ',' : '')) - } - elseif ($jsonParameterContentArray[$index] -match '(\s*)"\[.+\]"') { - # -and $jsonParameterContentArray[$index - 1] -like '*"value"*') { - # e.g. - # "policyDefinitionReferenceIds": { - # "value": [ - # "[reference(subscriptionResourceId('Microsoft.Authorization/policySetDefinitions', format('dep-#_namePrefix_#-polSet-{0}', parameters('serviceShort'))), '2021-06-01').policyDefinitions[0].policyDefinitionReferenceId]" - $prefix = $matches[1] - - $headerIndex = $index - while (($jsonParameterContentArray[$headerIndex] -notmatch '.+": (\{|\[)+' -or $jsonParameterContentArray[$headerIndex] -like '*"value"*') -and $headerIndex -gt -1) { - $headerIndex-- - } - - $value = (($jsonParameterContentArray[$headerIndex] -split ':')[0] -replace '"').Trim() - - $jsonParameterContentArray[$index] = ('{0}"<{1}>"{2}' -f $prefix, $value, ($jsonParameterContentArray[$index].Trim() -like '*,' ? ',' : '')) - } - } - $jsonParameterContent = $jsonParameterContentArray | Out-String - } - else { - # Case 2: Uses ARM-JSON parameter file - $jsonParameterContent = $rawContent.TrimEnd() - } - - # --------------------- # - # Add Bicep example # - # --------------------- # - if ($addBicep) { - - # [1/5] Get all parameters from the parameter object - $JSONParametersHashTable = (ConvertFrom-Json $jsonParameterContent -AsHashtable -Depth 99).parameters - - # [2/5] Handle the special case of Key Vault secret references (that have a 'reference' instead of a 'value' property) - # [2.1] Find all references and split them into managable objects - $keyVaultReferences = $JSONParametersHashTable.Keys | Where-Object { $JSONParametersHashTable[$_].Keys -contains 'reference' } - - if ($keyVaultReferences.Count -gt 0) { - $keyVaultReferenceData = @() - foreach ($reference in $keyVaultReferences) { - $resourceIdElem = $JSONParametersHashTable[$reference].reference.keyVault.id -split '/' - $keyVaultReferenceData += @{ - subscriptionId = $resourceIdElem[2] - resourceGroupName = $resourceIdElem[4] - vaultName = $resourceIdElem[-1] - secretName = $JSONParametersHashTable[$reference].reference.secretName - parameterName = $reference - } - } - } - - # [2.2] Remove any duplicates from the referenced key vaults and build 'existing' Key Vault references in Bicep format from them. - # Also, add a link to the corresponding Key Vault 'resource' to each identified Key Vault secret reference - $extendedKeyVaultReferences = @() - $counter = 0 - foreach ($reference in ($keyVaultReferenceData | Sort-Object -Property 'vaultName' -Unique)) { - $counter++ - $extendedKeyVaultReferences += @( - "resource kv$counter 'Microsoft.KeyVault/vaults@2019-09-01' existing = {", - (" name: '{0}'" -f $reference.vaultName), - (" scope: resourceGroup('{0}','{1}')" -f $reference.subscriptionId, $reference.resourceGroupName), - '}', - '' - ) - - # Add attribute for later correct reference - $keyVaultReferenceData | Where-Object { $_.vaultName -eq $reference.vaultName } | ForEach-Object { - $_['vaultResourceReference'] = "kv$counter" - } - } - - # [3/5] Replace all 'references' with the link to one of the 'existing' Key Vault resources - foreach ($parameterName in ($JSONParametersHashTable.Keys | Where-Object { $JSONParametersHashTable[$_].Keys -contains 'reference' })) { - $matchingTuple = $keyVaultReferenceData | Where-Object { $_.parameterName -eq $parameterName } - $JSONParametersHashTable[$parameterName] = "{0}.getSecret('{1}')" -f $matchingTuple.vaultResourceReference, $matchingTuple.secretName - } + # Build result + $testFilesContent += @( + '', + '

' + '' + 'via Bicep module' + '' + '```bicep', + ($formattedBicepExample | ForEach-Object { "$_" }).TrimEnd(), + '```', + '', + '
', + '

' + ) + } - # [4/5] Convert the JSON parameters to a Bicep parameters block - $conversionInputObject = @{ - JSONParameters = $JSONParametersHashTable - RequiredParametersList = $null -ne $RequiredParametersList ? $RequiredParametersList : @() - } - $bicepExample = ConvertTo-FormattedBicep @conversionInputObject + # -------------------- # + # Add JSON example # + # -------------------- # + if ($addJson) { - # [5/5] Create the final content block: That means - # - the 'existing' Key Vault resources - # - a 'module' header that mimics a module deployment - # - all parameters in Bicep format - $testFilesContent += @( - '', - '

' - '' - 'via Bicep module' - '' - '```bicep', - $extendedKeyVaultReferences, - "module $moduleNameCamelCase 'ts/modules:$(($FullModuleIdentifier -replace '\\|\/', '.').ToLower()):1.0.0 = {" - " name: '`${uniqueString(deployment().name)}-$moduleNamePascalCase'" - ' params: {' - $bicepExample.TrimEnd(), - ' }' - '}' - '```', - '', - '
' - '

' - ) + # [1/2] Get all parameters from the parameter object and order them recursively + $orderingInputObject = @{ + ParametersJSON = $paramsInJSONFormat | ConvertTo-Json -Depth 99 + RequiredParametersList = $RequiredParametersList } + $orderedJSONExample = Build-OrderedJSONObject @orderingInputObject - # -------------------- # - # Add JSON example # - # -------------------- # - if ($addJson) { - - # [1/2] Get all parameters from the parameter object and order them recursively - $orderingInputObject = @{ - ParametersJSON = (($jsonParameterContent | ConvertFrom-Json).parameters | ConvertTo-Json -Depth 99) - RequiredParametersList = $null -ne $RequiredParametersList ? $RequiredParametersList : @() - } - $orderedJSONExample = Build-OrderedJSONObject @orderingInputObject - - # [2/2] Create the final content block - $testFilesContent += @( - '', - '

', - '', - 'via JSON Parameter file', - '', - '```json', - $orderedJSONExample.TrimEnd(), - '```', - '', - '
' - '

' - ) - } + # [2/2] Create the final content block + $testFilesContent += @( + '', + '

' + '' + 'via JSON Parameter file' + '' + '```json', + $orderedJSONExample.Trim() + '```', + '', + '
', + '

' + ) } + $testFilesContent += @( '' ) $pathIndex++ } - foreach ($rawHeader in $usageExampleSectionHeaders) { $navigationHeader = (($rawHeader.header -replace '<\/?.+?>|[^A-Za-z0-9\s-]').Trim() -replace '\s+', '-').ToLower() # Remove any html and non-identifer elements $SectionContent += '- [{0}](#{1})' -f $rawHeader.title, $navigationHeader @@ -1496,8 +1315,7 @@ function Set-UsageExamplesSection { if ($PSCmdlet.ShouldProcess('Original file with new template references content', 'Merge')) { return Merge-FileWithNewContent -oldContent $ReadMeFileContent -newContent $SectionContent -SectionStartIdentifier $SectionStartIdentifier -ContentType 'nextH2' } - } - else { + } else { return $ReadMeFileContent } } @@ -1608,9 +1426,25 @@ function Initialize-ReadMe { $inTemplateResourceType = $formattedResourceType } + # Orphaned readme existing? + $orphanedReadMeFilePath = Join-Path (Split-Path $ReadMeFilePath -Parent) 'ORPHANED.md' + if (Test-Path $orphanedReadMeFilePath) { + $orphanedReadMeContent = Get-Content -Path $orphanedReadMeFilePath | ForEach-Object { "> $_" } + } + + # Moved readme existing? + $movedReadMeFilePath = Join-Path (Split-Path $ReadMeFilePath -Parent) 'MOVED-TO-AVM.md' + if (Test-Path $movedReadMeFilePath) { + $movedReadMeContent = Get-Content -Path $movedReadMeFilePath | ForEach-Object { "> $_" } + } + $initialContent = @( "# $moduleName ``[$inTemplateResourceType]``", '', + ((Test-Path $orphanedReadMeFilePath) ? $orphanedReadMeContent : $null), + ((Test-Path $orphanedReadMeFilePath) ? '' : $null), + ((Test-Path $movedReadMeFilePath) ? $movedReadMeContent : $null), + ((Test-Path $movedReadMeFilePath) ? '' : $null), $moduleDescription, '' '## Resource Types', @@ -1738,8 +1572,7 @@ function Set-ModuleReadMe { if (-not $TemplateFileContent) { if ((Split-Path -Path $TemplateFilePath -Extension) -eq '.bicep') { $templateFileContent = bicep build $TemplateFilePath --stdout | ConvertFrom-Json -AsHashtable - } - else { + } else { $templateFileContent = ConvertFrom-Json (Get-Content $TemplateFilePath -Encoding 'utf8' -Raw) -ErrorAction 'Stop' -AsHashtable } } @@ -1763,10 +1596,6 @@ function Set-ModuleReadMe { # Read original readme, if any. Then delete it to build from scratch if ((Test-Path $ReadMeFilePath) -and -not ([String]::IsNullOrEmpty((Get-Content $ReadMeFilePath -Raw)))) { $readMeFileContent = Get-Content -Path $ReadMeFilePath -Encoding 'utf8' - # Delete original readme - if ($PSCmdlet.ShouldProcess("File in path [$ReadMeFilePath]", 'Delete')) { - $null = Remove-Item $ReadMeFilePath -Force - } } # Make sure we preserve any manual notes a user might have added in the corresponding section if ($match = $readMeFileContent | Select-String -Pattern '## Notes') { @@ -1779,8 +1608,7 @@ function Set-ModuleReadMe { } $notes = $readMeFileContent[($startIndex - 1)..$endIndex] - } - else { + } else { $notes = @() } @@ -1855,7 +1683,6 @@ function Set-ModuleReadMe { } $readMeFileContent = Set-CrossReferencesSection @inputObject } - # Handle [Notes] section # ======================== if ($notes) { @@ -1876,8 +1703,15 @@ function Set-ModuleReadMe { Write-Verbose '============' Write-Verbose ($readMeFileContent | Out-String) - if ($PSCmdlet.ShouldProcess("File in path [$ReadMeFilePath]", 'Overwrite')) { - Set-Content -Path $ReadMeFilePath -Value $readMeFileContent -Force -Encoding 'utf8' + if (Test-Path $ReadMeFilePath) { + if ($PSCmdlet.ShouldProcess("File in path [$ReadMeFilePath]", 'Overwrite')) { + Set-Content -Path $ReadMeFilePath -Value $readMeFileContent -Force -Encoding 'utf8' + } Write-Verbose "File [$ReadMeFilePath] updated" -Verbose + } else { + if ($PSCmdlet.ShouldProcess("File in path [$ReadMeFilePath]", 'Create')) { + $null = New-Item -Path $ReadMeFilePath -Value $readMeFileContent -Force + } + Write-Verbose "File [$ReadMeFilePath] created" -Verbose } } diff --git a/avm/utilities/tools/Set-AVMModule.ps1 b/avm/utilities/tools/Set-AVMModule.ps1 index 6daa388dff..da272da232 100644 --- a/avm/utilities/tools/Set-AVMModule.ps1 +++ b/avm/utilities/tools/Set-AVMModule.ps1 @@ -76,10 +76,11 @@ function Set-AVMModule { [int] $Depth ) - # Load helper scripts + # # Load helper scripts . (Join-Path $PSScriptRoot 'helper' 'Set-ModuleFileAndFolderSetup.ps1') $resolvedPath = (Resolve-Path $ModuleFolderPath).Path + # Build up module file & folder structure if not yet existing. Should only run if an actual module path was provided (and not any of their parent paths) if (-not $SkipFileAndFolderSetup -and ((($resolvedPath -split '\bavm\b')[1].Trim('\,/') -split '[\/|\\]').Count -gt 2)) { if ($PSCmdlet.ShouldProcess("File & folder structure for path [$resolvedPath]", "Setup")) { @@ -88,7 +89,16 @@ function Set-AVMModule { } if ($Recurse) { - $relevantTemplatePaths = (Get-ChildItem -Path $resolvedPath -Recurse -File -Filter 'main.bicep').FullName + $childInput = @{ + Path = $resolvedPath + Recurse = $Recurse + File = $true + Filter = 'main.bicep' + } + if ($Depth) { + $childInput.Depth = $Depth + } + $relevantTemplatePaths = (Get-ChildItem @childInput).FullName } else { $relevantTemplatePaths = Join-Path $resolvedPath 'main.bicep' } @@ -101,13 +111,16 @@ function Set-AVMModule { # create reference as it must be loaded in the thread to work $ReadMeScriptFilePath = (Join-Path (Get-Item $PSScriptRoot).Parent.FullName 'pipelines' 'sharedScripts' 'Set-ModuleReadMe.ps1') + } else { + # Instatiate values to enable safe $using usage + $crossReferencedModuleList = $null + $ReadMeScriptFilePath = $null } # Using threading to speed up the process if ($PSCmdlet.ShouldProcess(('Building & generation of [{0}] modules in path [{1}]' -f $relevantTemplatePaths.Count, $resolvedPath), 'Execute')) { try { $job = $relevantTemplatePaths | ForEach-Object -ThrottleLimit $ThrottleLimit -AsJob -Parallel { - $resourceTypeIdentifier = 'avm-{0}' -f ($_ -split '[\/|\\]{1}avm[\/|\\]{1}(res|ptn)[\/|\\]{1}')[2] # avm/res// ############### @@ -151,7 +164,7 @@ function Set-AVMModule { # Clean up the job. $job | Remove-Job } finally { - # In case the user cancled the process, we need to make sure to stop all running jobs + # In case the user cancelled the process, we need to make sure to stop all running jobs $job | Remove-Job -Force -ErrorAction 'SilentlyContinue' } }