Provide support for Windows containers (#857)

* Rename Dockerfile

Signed-off-by: Tom Kerkhove <[email protected]>

* Introduce Windows image

* Change label of Windows stage

* Use correct Dockerfile

* Use nanoserver 1903

* Let's see what happens if we don't specify OS

* Switch back to one Dockerfile

* Code quality

* Fix volume?

* Fix volume mount

* Move to template

* Run Docker image

* Update changelog

* Update FAQ

* Update landing page

* Update runtime config

* Update deployment docs

* Further improve

* Remove template for now

* Remove file names

* Remove file names

* Improve tags

* Download variable

* List folders before reading

* Let's see

* Update scraper-ci.yml for Azure Pipelines

* Fix formatting

* Fix bad copy

* Fix path

* Get-Content

* Use powershell

* Cleanup

* Determine PR version old way

* Fix default image name

* Remove GitHub Package Registry for Windows

* Last Windows push

* Stamp folder for config based on OS in build

Signed-off-by: Tom Kerkhove <[email protected]>

* Read configuration folder from ENV

* Run Docker image with template (Windows)

* Run image via template only

* Revert local commit

* Use build Docker image template

* Let's see if this is valid

* Remove -

* Only build once

* Fix bad tag

* Remove trailing spaces

* Use conditions

* Refactor all the pipelines

* Update tags

* Provide test release build for Linux

* Update agents-scraper-release-official-test.yml for Azure Pipelines

* Add latest tag

* Change build job info for Linux

* Use YAML templates

* Use OS name variable to avoid duplication

* Add Windows

* Use script for determining version

* Add Windows stage

* Remove latest for windows

* Only comment for linux builds

* Reduce tags for Windows to OS-specific only

* Remove test build

* Push PR image without OS

* Don't comment for images with "-" in the tag

* Let's test other steps first

* Does this work?

* Update tagging strategy

* Let's see if casing is the issue

* Fix Markdown linting

* Cleanup, we're done!

* Emphasize multi-OS in image tagging strategy

* Add commented section for Windows

* Use latest .NET SDK

* Parameters

* Let's try default value

* Let's see if this works

* Fix bad template

* Fix checking parameters

* Assign to same variable

* Fix task display name

* Simplify again

build/azure-devops/agents-scraper-release-official.yml

@@ -34,22 +34,26 @@ stages:
        displayName: 'Determine ''{major}.{minor}'' version'
      - template: ./../templates/persist-variable.yml
        parameters:
-          variableName: 'Image.MajorMinorVersion'
+          currentVariableName: 'Image.MajorMinorVersion'
+          persistedVariableName: 'Image.MajorMinorVersion'
      - publish: $(Pipeline.Workspace)/variables
        artifact: variables
 - stage: Build
   dependsOn: Init
   displayName: Build and Push Docker image
   jobs:
-   - job: Build
+   - job: BuildLinux
+     displayName: Build Linux Docker image
      pool:
        vmImage: ubuntu-16.04
+     variables:
+       OS.Name: 'linux'
      steps:
      - download: current
        artifact: variables
      - template: ./../templates/read-variable.yml
        parameters:
-          variableName: 'Image.MajorMinorVersion'
+        variableName: 'Image.MajorMinorVersion'
      - task: DotNetCoreInstaller@0
        displayName: 'Install .NET SDK'
        inputs:
@@ -59,36 +63,77 @@ stages:
        inputs:
         command: test
         projects: src/Promitor.Scraper.Tests.Unit/Promitor.Scraper.Tests.Unit.csproj
-     - task: Docker@1
-       displayName: 'Build Docker image'
-       inputs:
-        dockerFile: ./src/Promitor.Scraper.Host/Dockerfile
-        arguments: '-t $(Image.Name):$(Image.MajorMinorVersion)'
+     - template: ./../templates/build-docker-image.yml
+       parameters:
         imageName: '$(Image.TaggedName)'
-        includeLatestTag: true
-        useDefaultContext: false
-        buildContext: ./src/
-     - task: Docker@1
-       displayName: 'Push ''{major}.{minor}.{patch}'' to Docker Hub'
-       inputs:
-        containerregistrytype: 'Container Registry'
-        dockerRegistryEndpoint: 'Docker Hub'
-        command: 'Push an image'
+        buildArgs: ''
+        tags: '--tag $(Image.Name):latest --tag $(Image.Name):$(Image.MajorMinorVersion) --tag $(Image.TaggedName)-$(OS.Name) --tag $(Image.Name):$(Image.MajorMinorVersion)-$(OS.Name) --tag $(Image.Name):latest-$(OS.Name)'
+        os: '$(OS.Name)'
+     - template: ./../templates/push-docker-image.yml
+       parameters:
+        imageDisplayName: '''{major}.{minor}.{patch}'''
         imageName: '$(Image.TaggedName)'
-     - task: Docker@1
-       displayName: 'Push ''{major}.{minor}'' to Docker Hub'
-       inputs:
-        containerregistrytype: 'Container Registry'
-        dockerRegistryEndpoint: 'Docker Hub'
-        command: 'Push an image'
+     - template: ./../templates/push-docker-image.yml
+       parameters:
+        imageDisplayName: '''{major}.{minor}.{patch}-$(OS.Name)'''
+        imageName: '$(Image.TaggedName)-$(OS.Name)'
+     - template: ./../templates/push-docker-image.yml
+       parameters:
+        imageDisplayName: '''{major}.{minor}'''
         imageName: '$(Image.Name):$(Image.MajorMinorVersion)'
-     - task: Docker@1
-       displayName: 'Push ''latest'' to Docker Hub'
-       inputs:
-        containerregistrytype: 'Container Registry'
-        dockerRegistryEndpoint: 'Docker Hub'
-        command: 'Push an image'
+     - template: ./../templates/push-docker-image.yml
+       parameters:
+        imageDisplayName: '''{major}.{minor}-$(OS.Name)'''
+        imageName: '$(Image.Name):$(Image.MajorMinorVersion)-$(OS.Name)'
+     - template: ./../templates/push-docker-image.yml
+       parameters:
+        imageDisplayName: '''latest'''
         imageName: '$(Image.Name):latest'
+     - template: ./../templates/push-docker-image.yml
+       parameters:
+        imageDisplayName: '''latest-$(OS.Name)'''
+        imageName: '$(Image.Name):latest-$(OS.Name)'
+   - job: BuildWindows
+     displayName: Build Windows Docker image
+     pool:
+       vmImage: windows-2019
+     variables:
+       OS.Name: 'windows'
+     steps:
+     - powershell: |
+        Write-Host "Determining '{major}.{minor}' for '$(Image.Version)'"
+        $index = "$(Image.Version)".IndexOf(".", 2);
+        $majorMinorVersion = "$(Image.Version)".Substring(0, $index);
+        Write-Host "Found version '$majorMinorVersion'"
+        Write-Output ("##vso[task.setvariable variable=Image.MajorMinorVersion;]$majorMinorVersion")
+       displayName: 'Determine ''{major}.{minor}'' version'
+     - task: DotNetCoreInstaller@0
+       displayName: 'Install .NET SDK'
+       inputs:
+        version: '$(DotNet.SDK.Version)'
+     - task: DotNetCoreCLI@2
+       displayName: 'Run Unit Tests'
+       inputs:
+        command: test
+        projects: src/Promitor.Scraper.Tests.Unit/Promitor.Scraper.Tests.Unit.csproj
+     - template: ./../templates/build-docker-image.yml
+       parameters:
+        imageName: '$(Image.TaggedName)-$(OS.Name)'
+        buildArgs: '--build-arg CONFIG_FOLDER="c:/config/"'
+        tags: '--tag $(Image.Name):$(Image.MajorMinorVersion)-$(OS.Name) --tag $(Image.Name):latest-$(OS.Name)'
+        os: '$(OS.Name)'
+     - template: ./../templates/push-docker-image.yml
+       parameters:
+        imageDisplayName: '''{major}.{minor}.{patch}-$(OS.Name)'''
+        imageName: '$(Image.TaggedName)-$(OS.Name)'
+     - template: ./../templates/push-docker-image.yml
+       parameters:
+        imageDisplayName: '''{major}.{minor}-$(OS.Name)'''
+        imageName: '$(Image.Name):$(Image.MajorMinorVersion)-$(OS.Name)'
+     - template: ./../templates/push-docker-image.yml
+       parameters:
+        imageDisplayName: '''latest-$(OS.Name)'''
+        imageName: '$(Image.Name):latest-$(OS.Name)'
 - stage: ReleaseGitHub
   displayName: Release on GitHub
   dependsOn: Build
@@ -102,7 +147,7 @@ stages:
        artifact: variables
      - template: ./../templates/read-variable.yml
        parameters:
-          variableName: 'Image.MajorMinorVersion'
+        variableName: 'Image.MajorMinorVersion'
      - task: GitHubRelease@0
        displayName: 'Create GitHub Release'
        inputs:
@@ -135,9 +180,9 @@ stages:
           ```
           ### Docker Image information
           New Docker image tags are available:
-          - `latest`
-          - `$(Image.Version)`
-          - `$(Image.MajorMinorVersion)`
+          - `latest`, `latest-linux`, `latest-windows`
+          - `$(Image.Version)`, `$(Image.Version)-linux`, `$(Image.Version)-windows`
+          - `$(Image.MajorMinorVersion)`, `$(Image.MajorMinorVersion)-linux`, `$(Image.MajorMinorVersion)-windows`
 
           Docker image is available on [Docker Hub](https://hub.docker.com/r/tomkerkhove/promitor-agent-scraper/).<br />
           For more information about our tagging strategy, feel free to read our [documentation](https://promitor.io/deployment/#image-tagging-strategy).

build/azure-devops/docs-ci.yml

@@ -36,9 +36,6 @@ stages:
          Write-Host "Build number is '$buildNumber'"
        displayName: 'Change Build Number to PR format'
        condition: eq(variables['Build.Reason'], 'PullRequest')
-     - template: ./../templates/persist-variable.yml
-       parameters:
-          variableName: 'Build.BuildNumber'
      - script: npm install
        displayName: 'Install npm Packages'
      - script: npm run --silent markdownlint

build/azure-devops/scraper-ci.yml

@@ -17,8 +17,8 @@ variables:
   DotNet.SDK.Version: '3.1.101'
   DotNet.Configuration: 'release'
   Image.Name: 'tomkerkhove/promitor-agent-scraper-ci'
-  Image.TaggedName: '$(Image.Name):$(Build.BuildNumber)'
-  Helm.Chart.Version: '0.0.0-$(Build.BuildNumber)'
+  Image.TaggedName: '$(Image.Name):PR$(Tags.PR)'
+  Helm.Chart.Version: '0.0.0-$(App.Version)'
   Helm.Chart.Name: 'promitor-agent-scraper-ci'
 stages:
 - stage: Init
@@ -42,7 +42,8 @@ stages:
        condition: eq(variables['Build.Reason'], 'PullRequest')
      - template: ./../templates/persist-variable.yml
        parameters:
-          variableName: 'Build.BuildNumber'
+        currentVariableName: 'Build.BuildNumber'
+        persistedVariableName: 'App.Version'
      - publish: $(Pipeline.Workspace)/variables
        artifact: variables
 - stage: Build
@@ -140,55 +141,102 @@ stages:
 - stage: Docker
   displayName: Docker Image
   dependsOn: [Init]
+  variables:
+    Tags.PR: '$(System.PullRequest.PullRequestNumber)-$(OS.Name)'
+    Tags.Experimental: 'experimental-$(OS.Name)'
   jobs:
-   - job: DockerBuild
-     displayName: Build Docker Image Tests
+   - job: DockerBuildLinux
+     displayName: Build Docker Image (Linux)
      condition: succeeded()
      pool:
        vmImage: ubuntu-16.04
+     variables:
+       Image.Experimental: 'docker.pkg.github.com/tomkerkhove/promitor/promitor-agent-scraper-ci:$(Tags.Experimental)'
+       Image.TaggedName.OSAgnostic: '$(Image.Name):pr$(System.PullRequest.PullRequestNumber)'
+       OS.Name: 'linux'
      steps:
      - download: current
        artifact: variables
      - template: ./../templates/read-variable.yml
        parameters:
-          variableName: 'Build.BuildNumber'
-     - task: Docker@1
-       displayName: 'Build an image'
-       inputs:
-         containerregistrytype: 'Container Registry'
-         dockerRegistryEndpoint: 'Docker Hub'
-         dockerFile: ./src/Promitor.Scraper.Host/Dockerfile
-         arguments: '--tag promitor-agent-scraper-ci --no-cache'
-         useDefaultContext: false
-         buildContext: ./src/
-         imageName: '$(Image.TaggedName)'
-         addDefaultLabels: false
-       condition: not(eq(variables['Build.Reason'], 'PullRequest'))
-     - task: Docker@1
-       displayName: 'Build an image'
-       inputs:
-         containerregistrytype: 'Container Registry'
-         dockerRegistryEndpoint: 'Docker Hub'
-         dockerFile: ./src/Promitor.Scraper.Host/Dockerfile
-         arguments: '--tag promitor-agent-scraper-ci --tag docker.pkg.github.com/tomkerkhove/promitor/promitor-agent-scraper-pr:$(System.PullRequest.PullRequestNumber) --no-cache'
-         useDefaultContext: false
-         buildContext: ./src/
-         imageName: '$(Image.TaggedName)'
-         addDefaultLabels: false
+        variableName: 'App.Version'
+     - ${{ if not(eq(variables['Build.Reason'], 'PullRequest')) }}:
+        - template: ./../templates/build-docker-image.yml
+          parameters:
+             imageName: '$(Image.TaggedName)'
+             tags: '--tag promitor-agent-scraper-ci'
+             os: '$(OS.Name)'
+     - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+        - template: ./../templates/build-docker-image.yml
+          parameters:
+             imageName: '$(Image.TaggedName)'
+             tags: '--tag promitor-agent-scraper-ci --tag $(Image.TaggedName.OSAgnostic) --tag docker.pkg.github.com/tomkerkhove/promitor/promitor-agent-scraper-pr:$(System.PullRequest.PullRequestNumber) --tag docker.pkg.github.com/tomkerkhove/promitor/promitor-agent-scraper-pr:$(Tags.PR)'
+             os: '$(OS.Name)'
+     - template: ./../templates/run-docker-image.yml
+       parameters:
+          volumes: '/src/metric-config.yaml:/config/metrics-declaration.yaml promitor-agent-scraper-ci'
+          imageName: '$(Image.TaggedName)'
+          os: '$(OS.Name)'
+     - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+        - template: ./../templates/push-docker-image.yml
+          parameters:
+              imageName: '$(Image.TaggedName)'
+        - template: ./../templates/push-docker-image.yml
+          parameters:
+              imageName: '$(Image.TaggedName.OSAgnostic)'
+        - template: ./../templates/push-docker-image.yml
+          parameters:
+              registryEndpoint: 'GitHub Package Registry - Docker (tomkerkhove)'
+              imageName: 'docker.pkg.github.com/tomkerkhove/promitor/promitor-agent-scraper-pr:$(System.PullRequest.PullRequestNumber)'
+        - template: ./../templates/push-docker-image.yml
+          parameters:
+              registryEndpoint: 'GitHub Package Registry - Docker (tomkerkhove)'
+              imageName: 'docker.pkg.github.com/tomkerkhove/promitor/promitor-agent-scraper-pr:$(Tags.PR)'
+     - ${{ if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/master')) }}:
+        - powershell: |
+            docker tag $(Image.TaggedName) $(Image.Experimental)
+          displayName: 'Tag image as experimental'
+          condition: and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
+        - template: ./../templates/push-docker-image.yml
+          parameters:
+              registryEndpoint: 'GitHub Package Registry - Docker (tomkerkhove)'
+              imageName: '$(Image.Experimental)'
+   - job: DockerBuildWindows
+     displayName: Build Docker Image (Windows)
+     condition: succeeded()
+     pool:
+       vmImage: windows-2019
+     variables:
+       OS.Name: 'windows'
+     steps:
+     - powershell: |
+         $prNumber = '$(System.PullRequest.PullRequestNumber)'
+         $buildNumber = '$(Build.BuildNumber)'
+         Write-Host "Initial build number: '$buildNumber'"
+         Write-Host "PR Number '$prNumber'"
+         $buildNumber = 'PR$(System.PullRequest.PullRequestNumber)'
+         Write-Host "##vso[build.updatebuildnumber]$buildNumber"
+         Write-Host "Build number is '$buildNumber'"
+       displayName: 'Change Build Number to PR format'
        condition: eq(variables['Build.Reason'], 'PullRequest')
-     - task: Docker@1
-       displayName: 'Run an image'
-       inputs:
-         containerregistrytype: 'Container Registry'
-         dockerRegistryEndpoint: 'Docker Hub'
-         command: 'Run an image'
-         imageName: '$(Image.TaggedName)'
-         containerName: 'promitor-ci'
-         ports: '8999:80'
-         volumes: '/src/metric-config.yaml:/config/metrics-declaration.yaml promitor-agent-scraper-ci'
-         envVars: |
-          PROMITOR_AUTH_APPID='$(Container.Env.AzureMonitor.Id)'
-          PROMITOR_AUTH_APPKEY='$(Container.Env.AzureMonitor.Key)'
+     - ${{ if not(eq(variables['Build.Reason'], 'PullRequest')) }}:
+        - template: ./../templates/build-docker-image.yml
+          parameters:
+             imageName: '$(Image.TaggedName)'
+             buildArgs: '--build-arg CONFIG_FOLDER="c:/config/"'
+             tags: '--tag promitor-agent-scraper-ci --tag promitor-agent-scraper-ci-non-pr'
+             os: '$(OS.Name)'
+     - template: ./../templates/build-docker-image.yml
+       parameters:
+          imageName: '$(Image.TaggedName)'
+          buildArgs: '--build-arg CONFIG_FOLDER="c:/config/"'
+          tags: '--tag promitor-agent-scraper-ci'
+          os: '$(OS.Name)'
+     - template: ./../templates/run-docker-image.yml
+       parameters:
+          volumes: '$(System.DefaultWorkingDirectory)\src:c:\config promitor-agent-scraper-ci'
+          imageName: '$(Image.TaggedName)'
+          os: '$(OS.Name)'
      - task: Docker@1
        displayName: 'Push ''$(Image.Name)'' image to Docker Hub'
        inputs:
@@ -197,26 +245,6 @@ stages:
          command: 'Push an image'
          imageName: '$(Image.TaggedName)'
        condition: eq(variables['Build.Reason'], 'PullRequest')
-     - task: Docker@1
-       displayName: 'Push PR image to GitHub Package Registry'
-       inputs:
-         containerregistrytype: 'Container Registry'
-         dockerRegistryEndpoint: 'GitHub Package Registry - Docker (tomkerkhove)'
-         command: 'Push an image'
-         imageName: 'docker.pkg.github.com/tomkerkhove/promitor/promitor-agent-scraper-pr:$(System.PullRequest.PullRequestNumber)'
-       condition: eq(variables['Build.Reason'], 'PullRequest')
-     - powershell: |
-         docker tag $(Image.TaggedName) docker.pkg.github.com/tomkerkhove/promitor/promitor-agent-scraper-ci:experimental
-       displayName: 'Tag image as experimental'
-       condition: and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
-     - task: Docker@1
-       displayName: 'Push experimental image to GitHub Package Registry'
-       inputs:
-         containerregistrytype: 'Container Registry'
-         dockerRegistryEndpoint: 'GitHub Package Registry - Docker (tomkerkhove)'
-         command: 'Push an image'
-         imageName: 'docker.pkg.github.com/tomkerkhove/promitor/promitor-agent-scraper-ci:experimental'
-       condition: and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
 - stage: Helm2
   displayName: Helm Chart (2.x)
   dependsOn: Docker
@@ -260,7 +288,7 @@ stages:
        artifact: variables
      - template: ./../templates/read-variable.yml
        parameters:
-          variableName: 'Build.BuildNumber'
+        variableName: 'App.Version'
      - powershell: |
         helm lint promitor-agent-scraper/ --strict
        workingDirectory: charts
@@ -270,7 +298,7 @@ stages:
        inputs:
         targetType: filePath
         filePath: ./build/helm/CI/Transform-Chart.ps1
-        arguments: '-chartName "$(Helm.Chart.Name)" -imageName "$(Image.Name)" -imageTag "$(Build.BuildNumber)"'
+        arguments: '-chartName "$(Helm.Chart.Name)" -imageName "$(Image.Name)" -imageTag "$(App.Version)"'
         workingDirectory: 'charts'
      - task: PowerShell@2
        displayName: 'Package Helm Chart'