diff --git a/.gitignore b/.gitignore index 348fa989287..e31e3bafd7e 100644 --- a/.gitignore +++ b/.gitignore @@ -460,6 +460,9 @@ src/dotnet/Mgmt.CI.BuildTools/NugetToolsPackage/CI.Tools.Package/build/tasks/net # ensure local debugging of the test-proxy will not end up committed tools/test-proxy/**/recordings/ +# ensure local build directory for test-proxy dockerfiles remains ignored +tools/test-proxy/docker/dev_certificate + # local settings .DS_Store .vscode diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index 4e0b0847cdb..4f31c92c3d2 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -44,3 +44,4 @@ $GetDocsMsMetadataForPackageFn = "Get-${Language}-DocsMsMetadataForPackage" $GetDocsMsDevLanguageSpecificPackageInfoFn = "Get-${Language}-DocsMsDevLanguageSpecificPackageInfo" $GetGithubIoDocIndexFn = "Get-${Language}-GithubIoDocIndex" $FindArtifactForApiReviewFn = "Find-${Language}-Artifacts-For-Apireview" +$TestProxyTrustCertFn = "Import-Dev-Cert-${Language}" diff --git a/eng/common/scripts/trust-proxy-certificate.ps1 b/eng/common/scripts/trust-proxy-certificate.ps1 new file mode 100644 index 00000000000..144d304cfd1 --- /dev/null +++ b/eng/common/scripts/trust-proxy-certificate.ps1 @@ -0,0 +1,6 @@ +. $PSScriptRoot/common.ps1 + +if ($TestProxyTrustCertFn -and (Test-Path "Function:$TestProxyTrustCertFn")) +{ + &$TestProxyTrustCertFn +} \ No newline at end of file diff --git a/tools/test-proxy/docker/dev_certificate/apply-dev-cert.sh b/eng/common/testproxy/apply-dev-cert.sh similarity index 100% rename from tools/test-proxy/docker/dev_certificate/apply-dev-cert.sh rename to eng/common/testproxy/apply-dev-cert.sh diff --git a/eng/common/testproxy/docker-start-proxy.ps1 b/eng/common/testproxy/docker-start-proxy.ps1 new file mode 100644 index 00000000000..4e5da37278a --- /dev/null +++ b/eng/common/testproxy/docker-start-proxy.ps1 @@ -0,0 +1,83 @@ + #!/usr/bin/env pwsh -c + +<# +.DESCRIPTION +Start the docker proxy container. If it is already running, quietly exit. Any other error should fail. +.PARAMETER Mode +"start" or "stop" to start up or stop the test-proxy instance. +.PARAMETER TargetFolder +The folder in which context the test proxy will be started. Defaults to current working directory. +#> +[CmdletBinding(SupportsShouldProcess = $true)] +param( + [ValidateSet("start", "stop")] + [String] + $Mode, + [String] + $TargetFolder = "." +) + +try { + docker --version | Out-Null +} +catch { + Write-Error "A invocation of docker --version failed. This indicates that docker is not properly installed or running." + Write-Error "Please check your docker invocation and try running the script again." +} + +$SELECTED_IMAGE_TAG = "1037115" +$CONTAINER_NAME = "ambitious_azsdk_test_proxy" +$LINUX_IMAGE_SOURCE = "azsdkengsys.azurecr.io/engsys/testproxy-lin:${SELECTED_IMAGE_TAG}" +$WINDOWS_IMAGE_SOURCE = "azsdkengsys.azurecr.io/engsys/testproxy-win:${SELECTED_IMAGE_TAG}" +$root = (Resolve-Path $TargetFolder).Path.Replace("`\", "/") + +function Get-Proxy-Container(){ + return (docker container ls -a --format "{{ json . }}" --filter "name=$CONTAINER_NAME" ` + | ConvertFrom-Json ` + | Select-Object -First 1) +} + + +$SelectedImage = $LINUX_IMAGE_SOURCE +$Initial = "" + +# most of the time, running this script on a windows machine will work just fine, as docker defaults to linux containers +# however, in CI, windows images default to _windows_ containers. We cannot swap them. We can tell if we're in a CI build by +# checking for the environment variable TF_BUILD. +if ($IsWindows -and $env:TF_BUILD){ + $SelectedImage = $WINDOWS_IMAGE_SOURCE + $Initial = "C:" +} + +if ($Mode -eq "start"){ + $proxyContainer = Get-Proxy-Container + + # if we already have one, we just need to check the state + if($proxyContainer){ + if ($proxyContainer.State -eq "running") + { + Write-Host "Discovered an already running instance of the test-proxy!. Exiting" + exit(0) + } + } + # else we need to create it + else { + Write-Host "Attempting creation of Docker host $CONTAINER_NAME" + Write-Host "docker container create -v `"${root}:${Initial}/etc/testproxy`" -p 5001:5001 -p 5000:5000 --name $CONTAINER_NAME $SelectedImage" + docker container create -v "${root}:${Initial}/etc/testproxy" -p 5001:5001 -p 5000:5000 --name $CONTAINER_NAME $SelectedImage + } + + Write-Host "Attempting start of Docker host $CONTAINER_NAME" + docker container start $CONTAINER_NAME +} + +if ($Mode -eq "stop"){ + $proxyContainer = Get-Proxy-Container + + if($proxyContainer){ + if($proxyContainer.State -eq "running"){ + Write-Host "Found a running instance of $CONTAINER_NAME, shutting it down." + docker container stop $CONTAINER_NAME + } + } +} \ No newline at end of file diff --git a/tools/test-proxy/docker/dev_certificate/dotnet-devcert.crt b/eng/common/testproxy/dotnet-devcert.crt similarity index 100% rename from tools/test-proxy/docker/dev_certificate/dotnet-devcert.crt rename to eng/common/testproxy/dotnet-devcert.crt diff --git a/tools/test-proxy/docker/dev_certificate/dotnet-devcert.pfx b/eng/common/testproxy/dotnet-devcert.pfx similarity index 100% rename from tools/test-proxy/docker/dev_certificate/dotnet-devcert.pfx rename to eng/common/testproxy/dotnet-devcert.pfx diff --git a/tools/test-proxy/docker/dev_certificate/localhost.conf b/eng/common/testproxy/localhost.conf similarity index 100% rename from tools/test-proxy/docker/dev_certificate/localhost.conf rename to eng/common/testproxy/localhost.conf diff --git a/eng/common/testproxy/test-proxy-docker.yml b/eng/common/testproxy/test-proxy-docker.yml new file mode 100644 index 00000000000..97617b6fd08 --- /dev/null +++ b/eng/common/testproxy/test-proxy-docker.yml @@ -0,0 +1,15 @@ +parameters: + rootFolder: '$(Build.SourcesDirectory)' + +steps: + - pwsh: | + $(Build.SourcesDirectory)/eng/common/scripts/trust-proxy-certificate.ps1 + displayName: 'Language Specific Certificate Trust' + + - pwsh: | + $(Build.SourcesDirectory)/eng/common/testproxy/docker-start-proxy.ps1 -Mode start -TargetFolder "${{ parameters.rootFolder }}" + displayName: 'Run the docker container' + + - pwsh: | + docker container ls -a + displayName: Check running container \ No newline at end of file diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml new file mode 100644 index 00000000000..9f24b0f0d52 --- /dev/null +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -0,0 +1,47 @@ +parameters: + rootFolder: '$(Build.SourcesDirectory)' + +steps: + - pwsh: | + $(Build.SourcesDirectory)/eng/common/scripts/trust-proxy-certificate.ps1 + displayName: 'Language Specific Certificate Trust' + + - pwsh: | + Write-Host "##vso[task.setvariable variable=OriginalPath]$env:PATH" + displayName: 'Store Path Value' + + - pwsh: | + Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Path]$(Build.SourcesDirectory)/eng/common/testproxy/dotnet-devcert.pfx" + Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Password]password" + displayName: 'Configure Kestrel Environment Variables' + + - task: UseDotNet@2 + displayName: "Use .NET Core SDK" + inputs: + packageType: sdk + version: 5.0.205 + + - pwsh: | + dotnet tool install azure.sdk.tools.testproxy ` + --tool-path $(Build.BinariesDirectory)/test-proxy ` + --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ` + --version 1.0.0-dev.20210811.2 + displayName: "Install test-proxy" + + - pwsh: | + Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe ` + -ArgumentList "--storage-location '${{ parameters.rootFolder }}'" ` + -NoNewWindow -PassThru + displayName: 'Run the testproxy - windows' + condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT')) + + # nohup does NOT continue beyond the current session if you use it within powershell + - bash: | + sudo nohup $(Build.BinariesDirectory)/test-proxy/test-proxy & + displayName: "Run the testproxy - linux/mac" + condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) + workingDirectory: "${{ parameters.rootFolder }}" + + - pwsh: | + Write-Host "##vso[task.setvariable variable=PATH]$(OriginalPath)" + displayName: 'Restore .NET version by resetting path' \ No newline at end of file diff --git a/eng/containers/ci.yml b/eng/containers/ci.yml index 60b18458307..73f50d2b920 100644 --- a/eng/containers/ci.yml +++ b/eng/containers/ci.yml @@ -5,6 +5,7 @@ parameters: - name: test_proxy_linux pool: 'ubuntu-20.04' dockerRepo: 'engsys/testproxy-lin' + prepareScript: tools/test-proxy/docker/prepare.ps1 dockerFile: 'tools/test-proxy/docker/dockerfile' stableTags: - 'latest' @@ -17,6 +18,7 @@ parameters: - name: test_proxy_windows pool: 'windows-2019' dockerRepo: 'engsys/testproxy-win' + prepareScript: tools/test-proxy/docker/prepare.ps1 dockerFile: 'tools/test-proxy/docker/dockerfile-win' stableTags: - 'latest' @@ -47,6 +49,11 @@ jobs: pool: vmImage: ${{ config.pool }} steps: + - ${{ if config.prepareScript }}: + - pwsh: | + ./${{ config.prepareScript }} + displayName: "Run prep script" + - task: Docker@2 displayName: Build ${{ config.name }}:$(imageTag) inputs: @@ -54,6 +61,7 @@ jobs: Dockerfile: ${{ config.dockerFile }} tags: $(imageTag) arguments: '-t $(containerRegistry).azurecr.io/${{ config.dockerRepo }}:$(imageTag)' + - task: Docker@2 displayName: Push ${{ config.name }}:$(imageTag) inputs: @@ -61,6 +69,7 @@ jobs: repository: ${{ config.dockerRepo }} command: push tags: $(imageTag) + - ${{ each stableTag in config.stableTags }}: - task: Docker@2 displayName: Build ${{ config.name }}:${{ stableTag }} @@ -69,6 +78,7 @@ jobs: Dockerfile: ${{ config.dockerFile }} tags: ${{ stableTag }} arguments: '-t $(containerRegistry).azurecr.io/${{ config.dockerRepo }}:${{ stableTag }}' + - task: Docker@2 displayName: Push ${{ config.name }}:${{ stableTag }} inputs: diff --git a/tools/http-fault-injector/README.md b/tools/http-fault-injector/README.md index 6b6b15f05de..3a66fd0f28f 100644 --- a/tools/http-fault-injector/README.md +++ b/tools/http-fault-injector/README.md @@ -11,7 +11,7 @@ 2. Install http-fault-injector ``` -> dotnet tool install azure.sdk.tools.httpfaultinjector --global --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk/nuget/v3/index.json +> dotnet tool install azure.sdk.tools.httpfaultinjector --global --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json You can invoke the tool using the following command: http-fault-injector Tool 'azure.sdk.tools.httpfaultinjector' (version '0.1.0') was successfully installed. @@ -19,7 +19,7 @@ Tool 'azure.sdk.tools.httpfaultinjector' (version '0.1.0') was successfully inst ## Updating ``` -> dotnet tool update azure.sdk.tools.httpfaultinjector --global --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk/nuget/v3/index.json +> dotnet tool update azure.sdk.tools.httpfaultinjector --global --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json Tool 'azure.sdk.tools.httpfaultinjector' was successfully updated from version '0.1.0' to version '0.1.1'. ``` diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md index b06395bcf61..a5c6e2e2909 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md @@ -10,7 +10,7 @@ For a detailed explanation, check the README.md one level up from this one. This 2. Install test-proxy ```powershell -> dotnet tool install azure.sdk.tools.testproxy --global --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk/nuget/v3/index.json +> dotnet tool install azure.sdk.tools.testproxy --global --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ``` This feed is available in [the public azure-sdk project.](https://dev.azure.com/azure-sdk/public/_packaging?_a=feed&feed=azure-sdk) @@ -290,7 +290,7 @@ The test-proxy server supports SSL, but due to its local-hosted nature, SSL vali Within this repository there is a single certificate. -* `dev_certificate/dotnet-devcert.pfx`: generated on a `Ubuntu` distribution using `openssl`. +* `eng/common/testproxy/dotnet-devcert.pfx`: generated on a `Ubuntu` distribution using `openssl`. Unfortunately, the `dotnet dev-certs` generated certificates are _not_ acceptable to a standard ubuntu distro. The issue is that the `KeyUsage` field in the `.crt` [MUST contain](https://github.com/dotnet/aspnetcore/issues/7246#issuecomment-541165030) the `keyCertSign` flag. Certificates generated by `dotnet dev-certs` do NOT have this flag. This means that if you're on Windows AND running the Ubuntu docker image, you will need to trust the `dotnet-devcert.pfx` locally prior to `docker run`. @@ -311,7 +311,7 @@ This will be automatically retrieved if you run the nuget installed version of t #### Option 2 -Import the appropriate already existing certificate within the `tools/test-proxy/docker/dev_certificate` folder. +Import the appropriate already existing certificate within the `eng/common/testproxy/` folder. ### Docker Image + SSL diff --git a/tools/test-proxy/docker/README.md b/tools/test-proxy/docker/README.md index 4c8246fa4ce..d9324fe3ca1 100644 --- a/tools/test-proxy/docker/README.md +++ b/tools/test-proxy/docker/README.md @@ -5,9 +5,17 @@ ### Build and Run -First, navigate to the folder containing the test-proxy dockerfile: `tools/test-proxy/docker`. +**Be aware there is a pre-step to prepare the working directory before building the dockerfile.** -Invoke to generate a container (with optional tag): +This is necessary to access supporting certificate files located in eng/common. There is intent to adjust this build upon a `test-assets image` that will allow us to relocate build context to this local folder, but that is as yet incomplete. + +Prior to any other steps, invoke: + +```pwsh +./prepare.ps1 +``` + +Then, invoke to generate a container (with optional tag): ```docker docker build . -t test-proxy diff --git a/tools/test-proxy/docker/dev_certificate/.gitattributes b/tools/test-proxy/docker/dev_certificate/.gitattributes deleted file mode 100644 index c5d9e95ea67..00000000000 --- a/tools/test-proxy/docker/dev_certificate/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -.sh eol=lf \ No newline at end of file diff --git a/tools/test-proxy/docker/dockerfile b/tools/test-proxy/docker/dockerfile index 8c79f4d5109..a86b88ec11a 100644 --- a/tools/test-proxy/docker/dockerfile +++ b/tools/test-proxy/docker/dockerfile @@ -28,8 +28,8 @@ RUN chmod +x $CERT_FOLDER/$CERT_IMPORT_SH \ # install the package RUN dotnet tool install azure.sdk.tools.testproxy \ --global \ - --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk/nuget/v3/index.json \ - --version 1.0.0-dev.20210729.1 \ + --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json \ + --version 1.0.0-dev.20210811.2 \ && mkdir /etc/testproxy EXPOSE 5001 diff --git a/tools/test-proxy/docker/dockerfile-win b/tools/test-proxy/docker/dockerfile-win index 6c4e4b3bb1d..79870dbdc39 100644 --- a/tools/test-proxy/docker/dockerfile-win +++ b/tools/test-proxy/docker/dockerfile-win @@ -83,8 +83,8 @@ RUN mkdir -p etc/testproxy # install the package RUN dotnet tool install azure.sdk.tools.testproxy \ --tool-path /proxyserver \ - --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk/nuget/v3/index.json \ - --version 1.0.0-dev.20210729.1 + --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json \ + --version 1.0.0-dev.20210811.2 EXPOSE 5001 EXPOSE 5000 diff --git a/tools/test-proxy/docker/prepare.ps1 b/tools/test-proxy/docker/prepare.ps1 new file mode 100644 index 00000000000..4bb36622853 --- /dev/null +++ b/tools/test-proxy/docker/prepare.ps1 @@ -0,0 +1,11 @@ +$copyDirectory = Resolve-Path (Join-Path -Path $PSScriptRoot -ChildPath "../../../eng/common/testproxy/") +$targetDirectory = "$PsScriptRoot/dev_certificate" + +if (-not (Test-Path $targetDirectory)) +{ + mkdir $targetDirectory +} + +# copy all files other than .yml from eng/common/scripts/testproxy into local directory dev_certificate +Get-ChildItem $copyDirectory -Exclude "*.yml" | % { Copy-Item -Path $_ -Destination "$targetDirectory/${$_.Name}" } + diff --git a/tools/test-proxy/documentation/trusting-cert-per-language.md b/tools/test-proxy/documentation/trusting-cert-per-language.md index 72eef7576fb..fbf170c2095 100644 --- a/tools/test-proxy/documentation/trusting-cert-per-language.md +++ b/tools/test-proxy/documentation/trusting-cert-per-language.md @@ -2,25 +2,25 @@ ## Generally -All necessary components for dev-certificate usage are present within the `dev_certificate` directory. +All necessary components for dev-certificate usage are present within the `eng/common/testproxy/` directory. **Note that this certificate was generated with password "password"** -Within are components of a **dev certificate** that has no usage outside of keeping your local usage of SSL happy. When running the container, you will need to trust this certificate (`dotnet-devcert.pfx`) if you want to connect to `https://localhost:5001` without cert validation failures. This certificate has no usage outside of your local box and is strictly associated with `CN=localhost`. +Within this folder are components of a **dev certificate** that has no usage outside of keeping your local usage of SSL happy. When running the container, you will need to trust `dotnet-devcert.pfx` if you want to connect to `https://localhost:5001` without cert validation failures. This certificate has no usage outside of your local box and is strictly associated with `CN=localhost`. ```powershell # ensure root access -> $rootCert = $(Import-PfxCertificate -FilePath ./dev_certificate/dotnet-devcert.pfx -CertStoreLocation 'Cert:\LocalMachine\Root') +> $rootCert = $(Import-PfxCertificate -FilePath eng/common/testproxy/dotnet-devcert.pfx -CertStoreLocation 'Cert:\LocalMachine\Root') ``` or via `dotnet` ```powershell -dotnet dev-certs https --clean --import ./dotnet-devcert.pfx --password="password" +dotnet dev-certs https --clean --import eng/common/testproxy/dotnet-devcert.pfx --password="password" dotnet dev-certs https --trust ``` -On a ubuntu-flavored distro of linux, feel free to re-use the import mechanism in the local file `tools/test-proxy/docker/dev_certificate/import-dev-cert.sh`. Prior to using locally, ensure $CERT_FOLDER environment variable is set to the local directory `dev_certificate` to access necessary files! +On a ubuntu-flavored distro of linux, feel free to re-use the import mechanism in the local file `eng/common/testproxy/import-dev-cert.sh`. Prior to using locally, ensure $CERT_FOLDER environment variable is set to the local directory containing the script. Otherwise it won't be able to access necessary files! Also note that taken to trust this cert will _also apply to installing the dotnet tool directly_. The test-proxy tool will consume the certificate just the same as the docker container does. diff --git a/tools/test-proxy/documentation/using-in-devops.md b/tools/test-proxy/documentation/using-in-devops.md index 120bdc7a2aa..e58ca8a61f9 100644 --- a/tools/test-proxy/documentation/using-in-devops.md +++ b/tools/test-proxy/documentation/using-in-devops.md @@ -40,7 +40,7 @@ Something along the lines of... dotnet tool install ` azure.sdk.tools.testproxy ` --tool-path $(Build.BinariesDirectory)/test-proxy ` - --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk/nuget/v3/index.json ` + --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ` --version displayName: "Install TestProxy"