From 087707403b0c8f4dc8b509c1cb1183ed3e4077f5 Mon Sep 17 00:00:00 2001 From: PREETI BANSAL Date: Sun, 13 Oct 2019 12:21:57 +0530 Subject: [PATCH] Adding support for pushing image in registry --- .../docker-common-v2/containerconnection.ts | 8 +- .../resources.resjson/en-US/resources.resjson | 28 +- Tasks/ContainerBuildV0/make.json | 9 +- Tasks/ContainerBuildV0/package-lock.json | 192 +++++++++++++ Tasks/ContainerBuildV0/package.json | 3 +- Tasks/ContainerBuildV0/src/buildcontainer.ts | 26 ++ Tasks/ContainerBuildV0/src/buildctl.ts | 93 +++---- Tasks/ContainerBuildV0/src/docker.ts | 32 +++ Tasks/ContainerBuildV0/src/dockerbuild.ts | 70 +++++ Tasks/ContainerBuildV0/src/dockerpush.ts | 253 ++++++++++++++++++ Tasks/ContainerBuildV0/src/utils.ts | 25 +- Tasks/ContainerBuildV0/task.json | 44 ++- Tasks/ContainerBuildV0/task.loc.json | 34 +-- 13 files changed, 692 insertions(+), 125 deletions(-) create mode 100644 Tasks/ContainerBuildV0/src/buildcontainer.ts create mode 100644 Tasks/ContainerBuildV0/src/docker.ts create mode 100644 Tasks/ContainerBuildV0/src/dockerbuild.ts create mode 100644 Tasks/ContainerBuildV0/src/dockerpush.ts diff --git a/Tasks/Common/docker-common-v2/containerconnection.ts b/Tasks/Common/docker-common-v2/containerconnection.ts index f276c1ecfa3d..4c11f74984e2 100644 --- a/Tasks/Common/docker-common-v2/containerconnection.ts +++ b/Tasks/Common/docker-common-v2/containerconnection.ts @@ -23,7 +23,13 @@ export default class ContainerConnection { private oldDockerConfigContent: string; constructor() { - this.dockerPath = tl.which("docker", true); + if(process.env["RUNNING_ON"] == "KUBERNETES") { + this.dockerPath = tl.which("img", true); + } + else { + this.dockerPath = tl.which("docker", true); + } + } public createCommand(): tr.ToolRunner { diff --git a/Tasks/ContainerBuildV0/Strings/resources.resjson/en-US/resources.resjson b/Tasks/ContainerBuildV0/Strings/resources.resjson/en-US/resources.resjson index 1f422fcda272..a4c9a89b854a 100644 --- a/Tasks/ContainerBuildV0/Strings/resources.resjson/en-US/resources.resjson +++ b/Tasks/ContainerBuildV0/Strings/resources.resjson/en-US/resources.resjson @@ -1,27 +1,27 @@ { "loc.friendlyName": "Container Build", - "loc.helpMarkDown": "[Learn more about this task](https://go.microsoft.com/fwlink/?linkid=851275) or [see the Kubernetes documentation](https://kubernetes.io/docs/home/)", + "loc.helpMarkDown": "[Learn more about this task](https://go.microsoft.com/fwlink/?linkid=2107300)", "loc.description": "Build Task", "loc.instanceNameFormat": "Container Build Task", - "loc.group.displayName.containerRepository": "Container Repository", - "loc.group.displayName.commands": "Commands", "loc.input.label.dockerRegistryServiceConnection": "Docker registry service connection", - "loc.input.help.dockerRegistryServiceConnection": "Select a Docker registry service connection. Required for commands that need to authenticate with a registry.", + "loc.input.help.dockerRegistryServiceConnection": "Select a Docker registry service connection.", "loc.input.label.repository": "Container repository", - "loc.input.help.repository": "Name of the repository.", + "loc.input.help.repository": "Name of the repository within the container registry.", "loc.input.label.Dockerfile": "Dockerfile", - "loc.input.help.Dockerfile": "Path to the Dockerfile.", + "loc.input.help.Dockerfile": "Path to Dockerfile.", "loc.input.label.buildContext": "Build context", - "loc.input.help.buildContext": "Path to the Build context.", + "loc.input.help.buildContext": "Path to Build context.", "loc.input.label.tags": "Tags", - "loc.input.help.tags": "A list of tags in separate lines. These tags are used in build, push and buildAndPush commands. Ex:

beta1.1
latest", + "loc.input.help.tags": "A list of tags in separate lines. Tags are used while building and pushing the image to container registry.", "loc.input.label.arguments": "Arguments", - "loc.input.help.arguments": "Docker command options. Ex:
For build command,
--build-arg HTTP_PROXY=http://10.20.30.2:1234 --quiet", - "loc.input.label.addPipelineData": "Add Pipeline metadata to image(s)", - "loc.input.help.addPipelineData": "By default pipeline data like source branch name, build id are added which helps with traceability. For example you can inspect an image to find out which pipeline built the image. You can opt out of this default behavior by using this input.", - "loc.messages.NotAValidSemverVersion": "Version not specified in correct format. E.g: 1.8.2, v1.8.2, 2.8.2, v2.8.2.", - "loc.messages.VerifyBuildctlInstallation": "Verifying Buildctl installation...", + "loc.input.help.arguments": "Arguments used while building the image.", + "loc.input.label.addPipelineMetadata": "Add Pipeline metadata to image(s)", + "loc.input.help.addPipelineMetadata": "By default pipeline data like source branch name, build id are added which helps with traceability. For example you can inspect an image to find out which pipeline built the image. You can opt out of this default behavior by using this input.", + "loc.messages.ContainerPatternNotFound": "No pattern found in Docker filepath parameter", "loc.messages.BuildctlLatestNotKnown": "Cannot get the latest Buildctl info from %s. Error %s. Using default Buildctl version %s.", "loc.messages.BuildctlDownloadFailed": "Failed to download Buildctl from location %s. Error %s", - "loc.messages.BuildctlNotFoundInFolder": "Buildctl executable not found in path %s" + "loc.messages.BuildctlNotFoundInFolder": "Buildctl executable not found in path %s", + "loc.messages.FileContentSynced": "Synced the file content to the disk. The content is %s.", + "loc.messages.VerifyBuildctlInstallation": "Verifying Buildctl installation...", + "loc.messages.WritingDockerConfigToTempFile": "Writing Docker config to temp file. File path: %s, Docker config: %s" } \ No newline at end of file diff --git a/Tasks/ContainerBuildV0/make.json b/Tasks/ContainerBuildV0/make.json index e6a3211cec8f..ff58e4b65b83 100644 --- a/Tasks/ContainerBuildV0/make.json +++ b/Tasks/ContainerBuildV0/make.json @@ -1,5 +1,11 @@ { "common": [ + { + "module": "../Common/docker-common-v2", + "type": "node", + "dest" : "./", + "compile" : true + }, { "module": "../Common/azure-arm-rest-v2", "type": "node", @@ -11,7 +17,8 @@ { "items": [ "node_modules/azure-arm-rest/node_modules/vsts-task-lib", - "node_modules/buildctl-common/node_modules/azure-pipelines-task-lib" + "node_modules/buildctl-common/node_modules/azure-pipelines-task-lib", + "node_modules/docker-common-v2/node_modules/azure-pipelines-task-lib" ], "options": "-Rf" } diff --git a/Tasks/ContainerBuildV0/package-lock.json b/Tasks/ContainerBuildV0/package-lock.json index 0b7846684a7d..a663b8b8e7f7 100644 --- a/Tasks/ContainerBuildV0/package-lock.json +++ b/Tasks/ContainerBuildV0/package-lock.json @@ -30,6 +30,24 @@ "@types/node": "*" } }, + "array-union": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz", + "integrity": "sha1-mjRBDk9OPaI96jdb5b5w8kd47Dk=", + "requires": { + "array-uniq": "^1.0.1" + } + }, + "array-uniq": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz", + "integrity": "sha1-r2rId6Jcx/dOBYiUdThY39sk/bY=" + }, + "arrify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=" + }, "azure-arm-rest-v2": { "version": "file:../../_build/Tasks/Common/azure-arm-rest-v2-2.0.0.tgz", "requires": { @@ -116,6 +134,36 @@ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, + "del": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/del/-/del-2.2.0.tgz", + "integrity": "sha1-mlDwS/NzJeKDtPROmFM2wlJFa9U=", + "requires": { + "globby": "^4.0.0", + "is-path-cwd": "^1.0.0", + "is-path-in-cwd": "^1.0.0", + "object-assign": "^4.0.1", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0", + "rimraf": "^2.2.8" + } + }, + "docker-common-v2": { + "version": "file:../../_build/Tasks/Common/docker-common-v2-1.0.0.tgz", + "requires": { + "azure-pipelines-task-lib": "2.8.0", + "del": "2.2.0", + "q": "1.4.1", + "vso-node-api": "6.0.1-preview" + }, + "dependencies": { + "q": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/q/-/q-1.4.1.tgz", + "integrity": "sha1-VXBbzZPF82c1MMLCy8DCs63cKG4=" + } + } + }, "ecdsa-sig-formatter": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", @@ -124,11 +172,76 @@ "safe-buffer": "^5.0.1" } }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + }, + "glob": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", + "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=", + "requires": { + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "2 || 3", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "globby": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-4.1.0.tgz", + "integrity": "sha1-CA9UVJ7BuCpsYOYx/ILhIR2+lfg=", + "requires": { + "array-union": "^1.0.1", + "arrify": "^1.0.0", + "glob": "^6.0.1", + "object-assign": "^4.0.1", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0" + } + }, "hoek": { "version": "2.16.3", "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz", "integrity": "sha1-ILt0A9POo5jpHcRxCo/xuCdKJe0=" }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "is-path-cwd": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-1.0.0.tgz", + "integrity": "sha1-0iXsIxMuie3Tj9p2dHLmLmXxEG0=" + }, + "is-path-in-cwd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-path-in-cwd/-/is-path-in-cwd-1.0.1.tgz", + "integrity": "sha512-FjV1RTW48E7CWM7eE/J2NJvAEEVektecDBVBE5Hh3nM1Jd0kvhHtX68Pr3xsDf857xt3Y4AkwVULK1Vku62aaQ==", + "requires": { + "is-path-inside": "^1.0.0" + } + }, + "is-path-inside": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-1.0.1.tgz", + "integrity": "sha1-jvW33lBDej/cprToZe96pVy0gDY=", + "requires": { + "path-is-inside": "^1.0.1" + } + }, "isemail": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/isemail/-/isemail-1.2.0.tgz", @@ -204,11 +317,75 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.3.tgz", "integrity": "sha1-cIFVpeROM/X9D8U+gdDUCpG+H/8=" }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "requires": { + "wrappy": "1" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + }, + "path-is-inside": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", + "integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM=" + }, + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=" + }, + "pinkie": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA=" + }, + "pinkie-promise": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "integrity": "sha1-ITXW36ejWMBprJsXh3YogihFD/o=", + "requires": { + "pinkie": "^2.0.0" + } + }, "q": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=" }, + "rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "requires": { + "glob": "^7.1.3" + }, + "dependencies": { + "glob": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", + "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + } + } + }, "safe-buffer": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz", @@ -261,6 +438,21 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" }, + "vso-node-api": { + "version": "6.0.1-preview", + "resolved": "https://registry.npmjs.org/vso-node-api/-/vso-node-api-6.0.1-preview.tgz", + "integrity": "sha1-RBprv5s8aNpiTbAeo1y6jwpMLKs=", + "requires": { + "q": "^1.0.1", + "tunnel": "0.0.4", + "underscore": "^1.8.3" + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + }, "xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/Tasks/ContainerBuildV0/package.json b/Tasks/ContainerBuildV0/package.json index 10bfdcf3017a..b6946437fba4 100644 --- a/Tasks/ContainerBuildV0/package.json +++ b/Tasks/ContainerBuildV0/package.json @@ -2,6 +2,7 @@ "dependencies": { "azure-arm-rest-v2": "file:../../_build/Tasks/Common/azure-arm-rest-v2-2.0.0.tgz", "azure-pipelines-task-lib": "2.8.0", - "azure-pipelines-tool-lib": "0.11.0" + "azure-pipelines-tool-lib": "0.11.0", + "docker-common-v2": "file:../../_build/Tasks/Common/docker-common-v2-1.0.0.tgz" } } diff --git a/Tasks/ContainerBuildV0/src/buildcontainer.ts b/Tasks/ContainerBuildV0/src/buildcontainer.ts new file mode 100644 index 000000000000..6e6a5d0a3dd1 --- /dev/null +++ b/Tasks/ContainerBuildV0/src/buildcontainer.ts @@ -0,0 +1,26 @@ +"use strict"; + +import tl = require('azure-pipelines-task-lib/task'); +import path = require('path'); +import docker = require("./docker"); +import buildctl = require("./buildctl"); + +tl.setResourcePath(path.join(__dirname, '..', 'task.json')); + +async function buildContainer() { + if(process.env["RUNNING_ON"] == "KUBERNETES") { + tl.debug("Building image using buildctl"); + buildctl.buildctlBuildAndPush(); + } + else { + tl.debug("Building image using docker"); + docker.dockerBuildAndPush(); + } +} + +buildContainer() + .then(() => { + tl.setResult(tl.TaskResult.Succeeded, ""); + }).catch((error) => { + tl.setResult(tl.TaskResult.Failed, error) + }); \ No newline at end of file diff --git a/Tasks/ContainerBuildV0/src/buildctl.ts b/Tasks/ContainerBuildV0/src/buildctl.ts index 68467dfc9d82..8cf8677c7a19 100644 --- a/Tasks/ContainerBuildV0/src/buildctl.ts +++ b/Tasks/ContainerBuildV0/src/buildctl.ts @@ -4,8 +4,9 @@ import tl = require('azure-pipelines-task-lib/task'); import path = require('path'); import * as toolLib from 'azure-pipelines-tool-lib/tool'; import utils = require("./utils"); - -tl.setResourcePath(path.join(__dirname, '..', 'task.json')); +import RegistryAuthenticationToken from "docker-common-v2/registryauthenticationprovider/registryauthenticationtoken"; +import ContainerConnection from "docker-common-v2/containerconnection"; +import { getDockerRegistryEndpointAuthenticationToken } from "docker-common-v2/registryauthenticationprovider/registryauthenticationtoken"; async function configureBuildctl() { var stableBuildKitVersion = await utils.getStableBuildctlVersion(); @@ -18,6 +19,7 @@ async function configureBuildctl() { } async function verifyBuildctl() { + await configureBuildctl(); tl.debug(tl.loc("VerifyBuildctlInstallation")); var buildctlToolPath = tl.which("buildctl", true); @@ -26,30 +28,38 @@ async function verifyBuildctl() { buildctlTool.arg("--help"); buildctlTool.exec(); } - -async function buildContainer() { - if(process.env["RUNNING_ON"] == "KUBERNETES") { - - tl.debug("Container building using buildctl"); - return buildUsingBuildctl(); - } - else { - - tl.debug("Container building using docker frontend"); - return buildUsingDocker(); - - } -} - -async function buildUsingBuildctl() { +export async function buildctlBuildAndPush() { await verifyBuildctl(); await utils.getBuildKitPod(); + let tags = tl.getDelimitedInput("tags", "\n"); + let endpointId = tl.getInput("dockerRegistryServiceConnection"); + let registryAuthenticationToken: RegistryAuthenticationToken = getDockerRegistryEndpointAuthenticationToken(endpointId); + + // Connect to any specified container registry + let connection = new ContainerConnection(); + connection.open(null, registryAuthenticationToken, true, false); + let repositoryName = tl.getInput("repository"); + if (!repositoryName) { + tl.warning("No repository is specified. Nothing will be pushed."); + } + + let imageNames: string[] = []; + if (tl.getInput("dockerRegistryServiceConnection")) { + let imageName = connection.getQualifiedImageName(repositoryName, true); + if (imageName) { + imageNames.push(imageName); + } + } + else { + imageNames = connection.getQualifiedImageNamesFromConfig(repositoryName, true); + } + var contextarg = "--local=context="+tl.getInput("buildContext", true); - var dockerfilearg = "--local=dockerfile="+tl.getInput("dockerFile", true); + var dockerfilearg = "--local=dockerfile="+tl.getInput("Dockerfile", true); var buildctlToolPath = tl.which("buildctl", true); var buildctlTool = tl.tool(buildctlToolPath); @@ -57,39 +67,14 @@ async function buildUsingBuildctl() { buildctlTool.arg('--frontend=dockerfile.v0'); buildctlTool.arg(contextarg); buildctlTool.arg(dockerfilearg); - - return buildctlTool.exec(); -} - -async function buildUsingDocker() { - - const dockerfilepath = tl.getInput("dockerFile", true); - const contextpath = tl.getInput("buildContext", true); - - var dockerToolPath = tl.which("docker", true); - var command = tl.tool(dockerToolPath); - - command.arg("build"); - command.arg(["-f", dockerfilepath]); - command.arg(contextpath); - - // setup variable to store the command output - let output = ""; - command.on("stdout", data => { - output += data; - }); - - let dockerHostVar = tl.getVariable("DOCKER_HOST"); - if (dockerHostVar) { - tl.debug(tl.loc('ConnectingToDockerHost', dockerHostVar)); + if (imageNames && imageNames.length > 0) { + imageNames.forEach(imageName => { + if (tags && tags.length > 0) { + tags.forEach(async tag => { + buildctlTool.arg(`--output=type=image,name=${imageName}:${tag},push=true`); + await buildctlTool.exec(); + }) + } + }) } - return command.exec(); -} - -configureBuildctl() - .then(() => buildContainer()) - .then(() => { - tl.setResult(tl.TaskResult.Succeeded, ""); - }).catch((error) => { - tl.setResult(tl.TaskResult.Failed, error) - }); \ No newline at end of file +} \ No newline at end of file diff --git a/Tasks/ContainerBuildV0/src/docker.ts b/Tasks/ContainerBuildV0/src/docker.ts new file mode 100644 index 000000000000..33b0e8ff98e2 --- /dev/null +++ b/Tasks/ContainerBuildV0/src/docker.ts @@ -0,0 +1,32 @@ +"use strict"; + +import path = require('path'); +import * as tl from "azure-pipelines-task-lib/task"; +import RegistryAuthenticationToken from "docker-common-v2/registryauthenticationprovider/registryauthenticationtoken"; +import ContainerConnection from "docker-common-v2/containerconnection"; +import { getDockerRegistryEndpointAuthenticationToken } from "docker-common-v2/registryauthenticationprovider/registryauthenticationtoken"; + +export async function dockerBuildAndPush() { + let endpointId = tl.getInput("dockerRegistryServiceConnection"); + let registryAuthenticationToken: RegistryAuthenticationToken = getDockerRegistryEndpointAuthenticationToken(endpointId); + + // Take the specified command + /*let command = tl.getInput("command", true).toLowerCase(); + let isLogout = (command === "logout");*/ + + // Connect to any specified container registry + let connection = new ContainerConnection(); + connection.open(null, registryAuthenticationToken, true, false); + + let resultPaths = ""; + /* tslint:disable:no-var-requires */ + let commandImplementation = require("./dockerbuild"); + await commandImplementation.runBuild(connection, (pathToResult) => { + resultPaths += pathToResult; + }) + + commandImplementation = require("./dockerpush") + await commandImplementation.run(connection, (pathToResult) => { + resultPaths += pathToResult; + }) +} diff --git a/Tasks/ContainerBuildV0/src/dockerbuild.ts b/Tasks/ContainerBuildV0/src/dockerbuild.ts new file mode 100644 index 000000000000..981603ea6584 --- /dev/null +++ b/Tasks/ContainerBuildV0/src/dockerbuild.ts @@ -0,0 +1,70 @@ +"use strict"; + +import * as fs from "fs"; +import * as path from "path"; +import * as tl from "azure-pipelines-task-lib/task"; +import ContainerConnection from "docker-common-v2/containerconnection"; +import * as dockerCommandUtils from "docker-common-v2/dockercommandutils"; +import * as fileUtils from "docker-common-v2/fileutils"; +import * as pipelineUtils from "docker-common-v2/pipelineutils"; +import * as utils from "./utils"; + +export async function runBuild(connection: ContainerConnection, outputUpdate: (data: string) => any, isBuildAndPushCommand?: boolean) { + // find dockerfile path + let dockerfilepath = tl.getInput("Dockerfile", true); + let dockerFile = fileUtils.findDockerFile(dockerfilepath); + + if(!tl.exist(dockerFile)) { + throw new Error(tl.loc('ContainerDockerFileNotFound', dockerfilepath)); + } + + // get command arguments + // ignore the arguments input if the command is buildAndPush, as it is ambiguous + let commandArguments = isBuildAndPushCommand ? "" : dockerCommandUtils.getCommandArguments(tl.getInput("arguments", false)); + + // get qualified image names by combining container registry(s) and repository + let repositoryName = tl.getInput("repository"); + let imageNames: string[] = []; + // if container registry is provided, use that + // else, use the currently logged in registries + if (tl.getInput("dockerRegistryServiceConnection")) { + let imageName = connection.getQualifiedImageName(repositoryName, true); + if (imageName) { + imageNames.push(imageName); + } + } + else { + imageNames = connection.getQualifiedImageNamesFromConfig(repositoryName, true); + } + + const addPipelineData = tl.getBoolInput("addPipelineMetadata"); + // get label arguments + let labelArguments = pipelineUtils.getDefaultLabels(addPipelineData); + + // get tags input + let tags = tl.getDelimitedInput("tags", "\n"); + let tagArguments: string[] = []; + // find all the tag arguments to be added to the command + if (imageNames && imageNames.length > 0) { + imageNames.forEach(imageName => { + if (tags && tags.length > 0) { + tags.forEach(tag => { + tagArguments.push(imageName + ":" + tag); + }); + } + else { + // pass just the imageName and not the tag. This will tag the image with latest tag as per the default behavior of the build command. + tagArguments.push(imageName); + } + }); + } + else { + tl.debug(tl.loc('NotAddingAnyTagsToBuild')); + } + + let output = ""; + return dockerCommandUtils.build(connection, dockerFile, commandArguments, labelArguments, tagArguments, (data) => output += data).then(() => { + let taskOutputPath = utils.writeTaskOutput("build", output); + outputUpdate(taskOutputPath); + }); +} diff --git a/Tasks/ContainerBuildV0/src/dockerpush.ts b/Tasks/ContainerBuildV0/src/dockerpush.ts new file mode 100644 index 000000000000..5fde3a7129eb --- /dev/null +++ b/Tasks/ContainerBuildV0/src/dockerpush.ts @@ -0,0 +1,253 @@ +"use strict"; + +import * as tl from "azure-pipelines-task-lib/task"; +import * as fs from 'fs'; +import ContainerConnection from "docker-common-v2/containerconnection"; +import * as dockerCommandUtils from "docker-common-v2/dockercommandutils"; +import * as utils from "./utils"; +import { findDockerFile } from "docker-common-v2/fileutils"; +import { WebRequest, WebResponse, sendRequest } from "azure-arm-rest-v2/webClient"; +import { getBaseImageName, getResourceName, getBaseImageNameFromDockerFile } from "docker-common-v2/containerimageutils"; +import * as pipelineUtils from "docker-common-v2/pipelineutils"; + +import Q = require('q'); + +const matchPatternForDigestAndSize = new RegExp(/sha256\:([\w]+)(\s+)size\:\s([\w]+)/); + +function pushMultipleImages(connection: ContainerConnection, imageNames: string[], tags: string[], commandArguments: string, onCommandOut: (image, output) => any): any { + let promise: Q.Promise; + // create chained promise of push commands + if (imageNames && imageNames.length > 0) { + imageNames.forEach(imageName => { + if (tags && tags.length > 0) { + tags.forEach(tag => { + let imageNameWithTag = imageName + ":" + tag; + tl.debug("Pushing ImageNameWithTag: " + imageNameWithTag); + if (promise) { + promise = promise.then(() => { + return dockerCommandUtils.push(connection, imageNameWithTag, commandArguments, onCommandOut) + }); + } + else { + promise = dockerCommandUtils.push(connection, imageNameWithTag, commandArguments, onCommandOut); + } + }); + } + else { + tl.debug("Pushing ImageName: " + imageName); + if (promise) { + promise = promise.then(() => { + return dockerCommandUtils.push(connection, imageName, commandArguments, onCommandOut) + }); + } + else { + promise = dockerCommandUtils.push(connection, imageName, commandArguments, onCommandOut); + } + } + }); + } + + // will return undefined promise in case imageNames is null or empty list + return promise; +} + +export async function run(connection: ContainerConnection, outputUpdate: (data: string) => any, isBuildAndPushCommand?: boolean) { + // ignore the arguments input if the command is buildAndPush, as it is ambiguous + let commandArguments = isBuildAndPushCommand ? "" : dockerCommandUtils.getCommandArguments(tl.getInput("arguments", false)); + + // get tags input + let tags = tl.getDelimitedInput("tags", "\n"); + + // get repository input + let repositoryName = tl.getInput("repository"); + if (!repositoryName) { + tl.warning("No repository is specified. Nothing will be pushed."); + } + + let imageNames: string[] = []; + // if container registry is provided, use that + // else, use the currently logged in registries + if (tl.getInput("dockerRegistryServiceConnection")) { + let imageName = connection.getQualifiedImageName(repositoryName, true); + if (imageName) { + imageNames.push(imageName); + } + } + else { + imageNames = connection.getQualifiedImageNamesFromConfig(repositoryName, true); + } + + const dockerfilepath = tl.getInput("Dockerfile", true); + let dockerFile = ""; + if (isBuildAndPushCommand) { + // For buildAndPush command, to find out the base image name, we can use the + // Dockerfile returned by findDockerfile as we are sure that this is used + // for building. + dockerFile = findDockerFile(dockerfilepath); + if (!tl.exist(dockerFile)) { + throw new Error(tl.loc('ContainerDockerFileNotFound', dockerfilepath)); + } + } + + // push all tags + let output = ""; + let outputImageName = ""; + let digest = ""; + let imageSize = ""; + let promise = pushMultipleImages(connection, imageNames, tags, commandArguments, (image, commandOutput) => { + output += commandOutput; + outputImageName = image; + let digest = extractDigestFromOutput(commandOutput, matchPatternForDigestAndSize); + tl.debug("outputImageName: " + outputImageName + "\n" + "commandOutput: " + commandOutput + "\n" + "digest:" + digest + "imageSize:" + imageSize); + publishToImageMetadataStore(connection, outputImageName, tags, digest, dockerFile).then((result) => { + tl.debug("ImageDetailsApiResponse: " + JSON.stringify(result)); + }, (error) => { + tl.warning("publishToImageMetadataStore failed with error: " + error); + }); + }); + + if (promise) { + promise = promise.then(() => { + let taskOutputPath = utils.writeTaskOutput("push", output); + outputUpdate(taskOutputPath); + }); + } + else { + tl.debug(tl.loc('NotPushingAsNoLoginFound')); + promise = Q.resolve(null); + } + + return promise; +} + +async function publishToImageMetadataStore(connection: ContainerConnection, imageName: string, tags: string[], digest: string, dockerFilePath: string): Promise { + // Getting imageDetails + const imageUri = getResourceName(imageName, digest); + const baseImageName = dockerFilePath ? getBaseImageNameFromDockerFile(dockerFilePath) : "NA"; + const history = await dockerCommandUtils.getHistory(connection, imageName); + if (!history) { + return null; + } + + const layers = dockerCommandUtils.getLayers(history); + const imageSize = dockerCommandUtils.getImageSize(layers); + + // Get data for ImageFingerPrint + // v1Name is the layerID for the final layer in the image + // v2Blobs are ordered list of layers that represent the given image, obtained from docker inspect output + const v1Name = dockerCommandUtils.getImageFingerPrintV1Name(history); + const imageRootfsLayers = await dockerCommandUtils.getImageRootfsLayers(connection, v1Name); + let imageFingerPrint: { [key: string]: string | string[] } = {}; + if (imageRootfsLayers && imageRootfsLayers.length > 0) { + imageFingerPrint = dockerCommandUtils.getImageFingerPrint(imageRootfsLayers, v1Name); + } + + const addPipelineData = tl.getBoolInput("addPipelineMetadata"); + + // Getting pipeline variables + const build = "build"; + const hostType = tl.getVariable("System.HostType").toLowerCase(); + const runId = hostType === build ? parseInt(tl.getVariable("Build.BuildId")) : parseInt(tl.getVariable("Release.ReleaseId")); + const pipelineVersion = addPipelineData ? hostType === build ? tl.getVariable("Build.BuildNumber") : tl.getVariable("Release.ReleaseName") : ""; + const pipelineName = addPipelineData ? tl.getVariable("System.DefinitionName") : ""; + const pipelineId = addPipelineData ? tl.getVariable("System.DefinitionId") : ""; + const jobName = addPipelineData ? tl.getVariable("System.PhaseDisplayName") : ""; + const creator = addPipelineData ? dockerCommandUtils.getCreatorEmail() : ""; + const logsUri = addPipelineData ? dockerCommandUtils.getPipelineLogsUrl() : ""; + const artifactStorageSourceUri = addPipelineData ? dockerCommandUtils.getPipelineUrl() : ""; + + const repoUrl = tl.getVariable("Build.Repository.Uri"); + const contextUrl = addPipelineData && repoUrl ? repoUrl : ""; + + const commitId = tl.getVariable("Build.SourceVersion"); + const revisionId = addPipelineData && commitId ? commitId : ""; + + const labelArguments = pipelineUtils.getDefaultLabels(addPipelineData); + const buildOptions = dockerCommandUtils.getBuildAndPushArguments(dockerFilePath, labelArguments, tags); + + // Capture Repository data for Artifact traceability + const repositoryTypeName = tl.getVariable("Build.Repository.Provider"); + const repositoryId = tl.getVariable("Build.Repository.ID"); + const repositoryName = tl.getVariable("Build.Repository.Name"); + const branch = tl.getVariable("Build.SourceBranchName"); + + const requestUrl = tl.getVariable("System.TeamFoundationCollectionUri") + tl.getVariable("System.TeamProject") + "/_apis/deployment/imagedetails?api-version=5.0-preview.1"; + const requestBody: string = JSON.stringify( + { + "imageName": imageUri, + "imageUri": imageUri, + "hash": digest, + "baseImageName": baseImageName, + "distance": layers.length, + "imageType": "", + "mediaType": "", + "tags": tags, + "layerInfo": layers, + "runId": runId, + "pipelineVersion": pipelineVersion, + "pipelineName": pipelineName, + "pipelineId": pipelineId, + "jobName": jobName, + "imageSize": imageSize, + "creator": creator, + "logsUri": logsUri, + "artifactStorageSourceUri": artifactStorageSourceUri, + "contextUrl": contextUrl, + "revisionId": revisionId, + "buildOptions": buildOptions, + "repositoryTypeName": repositoryTypeName, + "repositoryId": repositoryId, + "repositoryName": repositoryName, + "branch": branch, + "imageFingerPrint": imageFingerPrint + } + ); + + return sendRequestToImageStore(requestBody, requestUrl); +} + +function extractDigestFromOutput(dockerPushCommandOutput: string, matchPattern: RegExp): string { + // SampleCommandOutput : The push refers to repository [xyz.azurecr.io/acr-helloworld] + // 3b7670606102: Pushed + // e2af85e4b310: Pushed ce8609e9fdad: Layer already exists + // f2b18e6d6636: Layer already exists + // 62: digest: sha256:5e3c9cf1692e129744fe7db8315f05485c6bb2f3b9f6c5096ebaae5d5bfbbe60 size: 5718 + + // Below regex will extract part after sha256, so expected return value will be 5e3c9cf1692e129744fe7db8315f05485c6bb2f3b9f6c5096ebaae5d5bfbbe60 + const imageMatch = dockerPushCommandOutput.match(matchPattern); + let digest = ""; + if (imageMatch && imageMatch.length >= 1) { + digest = imageMatch[1]; + } + + return digest; +} + +async function sendRequestToImageStore(requestBody: string, requestUrl: string): Promise { + const request = new WebRequest(); + const accessToken: string = tl.getEndpointAuthorizationParameter('SYSTEMVSSCONNECTION', 'ACCESSTOKEN', false); + request.uri = requestUrl; + request.method = 'POST'; + request.body = requestBody; + request.headers = { + "Content-Type": "application/json", + "Authorization": "Bearer " + accessToken + }; + + tl.debug("requestUrl: " + requestUrl); + tl.debug("requestBody: " + requestBody); + tl.debug("accessToken: " + accessToken); + + try { + tl.debug("Sending request for pushing image to Image meta data store"); + const response = await sendRequest(request); + return response; + } + catch (error) { + tl.debug("Unable to push to Image Details Artifact Store, Error: " + error); + } + + return Promise.resolve(); +} + + diff --git a/Tasks/ContainerBuildV0/src/utils.ts b/Tasks/ContainerBuildV0/src/utils.ts index d9b367bccc0c..78f8e1b5b581 100644 --- a/Tasks/ContainerBuildV0/src/utils.ts +++ b/Tasks/ContainerBuildV0/src/utils.ts @@ -7,6 +7,7 @@ import fs = require('fs'); import webclient = require("azure-arm-rest-v2/webClient"); import * as os from "os"; import * as util from "util"; +import * as fileutils from "docker-common-v2/fileutils"; const buildctlToolName = "buildctl" const uuidV4 = require('uuid/v4'); @@ -31,6 +32,7 @@ export async function getStableBuildctlVersion(): Promise { return stableBuildctlVersion; } + export async function downloadBuildctl(version: string): Promise { let buildctlDownloadPath: string = null; @@ -102,7 +104,7 @@ export async function getBuildKitPod() { let request = new webclient.WebRequest(); let headers = { - "key": tl.getVariable('Build.Repository.Name')+tl.getInput("dockerFile", true) + "key": tl.getVariable('Build.Repository.Name')+tl.getInput("Dockerfile", true) }; let webRequestOptions:webclient.WebRequestOptions = {retriableErrorCodes: [], retriableStatusCodes: [], retryCount: 1, retryIntervalInSeconds: 5, retryRequestTimedout: true}; @@ -135,4 +137,25 @@ function getArchiveExtension(): string { return ".zip"; } return ".tar.gz"; +} + +function getTaskOutputDir(command: string): string { + let tempDirectory = tl.getVariable('agent.tempDirectory') || os.tmpdir(); + let taskOutputDir = path.join(tempDirectory, "task_outputs"); + return taskOutputDir; +} + +export function writeTaskOutput(commandName: string, output: string): string { + let taskOutputDir = getTaskOutputDir(commandName); + if (!fs.existsSync(taskOutputDir)) { + fs.mkdirSync(taskOutputDir); + } + + let outputFileName = commandName + "_" + Date.now() + ".txt"; + let taskOutputPath = path.join(taskOutputDir, outputFileName); + if (fileutils.writeFileSync(taskOutputPath, output) == 0) { + tl.warning(tl.loc('NoDataWrittenOnFile', taskOutputPath)); + } + + return taskOutputPath; } \ No newline at end of file diff --git a/Tasks/ContainerBuildV0/task.json b/Tasks/ContainerBuildV0/task.json index 2d32e98c3036..9e175b452505 100644 --- a/Tasks/ContainerBuildV0/task.json +++ b/Tasks/ContainerBuildV0/task.json @@ -4,7 +4,7 @@ "friendlyName": "Container Build", "description": "Build Task", "helpUrl": "https://docs.microsoft.com/azure/devops/pipelines/tasks", - "helpMarkDown": "[Learn more about this task](https://go.microsoft.com/fwlink/?linkid=851275) or [see the Kubernetes documentation](https://kubernetes.io/docs/home/)", + "helpMarkDown": "[Learn more about this task](https://go.microsoft.com/fwlink/?linkid=2107300)", "category": "Build", "visibility": [ "Build", @@ -20,34 +20,21 @@ "satisfies": [ "Buildctl" ], - "groups": [ - { - "name": "containerRepository", - "displayName": "Container Repository", - "isExpanded": true - }, - { - "name": "commands", - "displayName": "Commands", - "isExpanded": true - } - ], + "groups": [], "inputs": [ { "name": "dockerRegistryServiceConnection", "type": "connectedService:dockerregistry", "label": "Docker registry service connection", - "groupName": "containerRepository", - "helpMarkDown": "Select a Docker registry service connection. Required for commands that need to authenticate with a registry." + "helpMarkDown": "Select a Docker registry service connection." }, { "name": "repository", "label": "Container repository", "type": "string", - "helpMarkDown": "Name of the repository.", + "helpMarkDown": "Name of the repository within the container registry.", "defaultValue": "", "visibleRule": "command != login && command != logout", - "groupName": "containerRepository", "properties": { "EditableOptions": "True" } @@ -58,14 +45,14 @@ "label": "Dockerfile", "defaultValue": ".", "required": true, - "helpMarkDown": "Path to the Dockerfile." + "helpMarkDown": "Path to Dockerfile." }, { "name": "buildContext", "type": "filePath", "label": "Build context", "defaultValue": ".", - "helpMarkDown": "Path to the Build context." + "helpMarkDown": "Path to Build context." }, { "name": "tags", @@ -76,21 +63,18 @@ "rows": "2" }, "label": "Tags", - "groupName": "commands", - "helpMarkDown": "A list of tags in separate lines. These tags are used in build, push and buildAndPush commands. Ex:

beta1.1
latest" + "helpMarkDown": "A list of tags in separate lines. Tags are used while building and pushing the image to container registry." }, { "name": "arguments", "type": "string", "label": "Arguments", - "groupName": "commands", - "helpMarkDown": "Docker command options. Ex:
For build command,
--build-arg HTTP_PROXY=http://10.20.30.2:1234 --quiet" + "helpMarkDown": "Arguments used while building the image." }, { - "name": "addPipelineData", + "name": "addPipelineMetadata", "type": "boolean", "label": "Add Pipeline metadata to image(s)", - "groupName": "commands", "defaultValue": "true", "helpMarkDown": "By default pipeline data like source branch name, build id are added which helps with traceability. For example you can inspect an image to find out which pipeline built the image. You can opt out of this default behavior by using this input." } @@ -98,14 +82,16 @@ "instanceNameFormat": "Container Build Task", "execution": { "Node": { - "target": "src//buildctl.js" + "target": "src//buildcontainer.js" } }, "messages": { - "NotAValidSemverVersion": "Version not specified in correct format. E.g: 1.8.2, v1.8.2, 2.8.2, v2.8.2.", - "VerifyBuildctlInstallation": "Verifying Buildctl installation...", + "ContainerPatternNotFound": "No pattern found in Docker filepath parameter", "BuildctlLatestNotKnown": "Cannot get the latest Buildctl info from %s. Error %s. Using default Buildctl version %s.", "BuildctlDownloadFailed": "Failed to download Buildctl from location %s. Error %s", - "BuildctlNotFoundInFolder": "Buildctl executable not found in path %s" + "BuildctlNotFoundInFolder": "Buildctl executable not found in path %s", + "FileContentSynced": "Synced the file content to the disk. The content is %s.", + "VerifyBuildctlInstallation": "Verifying Buildctl installation...", + "WritingDockerConfigToTempFile": "Writing Docker config to temp file. File path: %s, Docker config: %s" } } diff --git a/Tasks/ContainerBuildV0/task.loc.json b/Tasks/ContainerBuildV0/task.loc.json index d6ef2e472af3..38c68861dccd 100644 --- a/Tasks/ContainerBuildV0/task.loc.json +++ b/Tasks/ContainerBuildV0/task.loc.json @@ -20,24 +20,12 @@ "satisfies": [ "Buildctl" ], - "groups": [ - { - "name": "containerRepository", - "displayName": "ms-resource:loc.group.displayName.containerRepository", - "isExpanded": true - }, - { - "name": "commands", - "displayName": "ms-resource:loc.group.displayName.commands", - "isExpanded": true - } - ], + "groups": [], "inputs": [ { "name": "dockerRegistryServiceConnection", "type": "connectedService:dockerregistry", "label": "ms-resource:loc.input.label.dockerRegistryServiceConnection", - "groupName": "containerRepository", "helpMarkDown": "ms-resource:loc.input.help.dockerRegistryServiceConnection" }, { @@ -47,7 +35,6 @@ "helpMarkDown": "ms-resource:loc.input.help.repository", "defaultValue": "", "visibleRule": "command != login && command != logout", - "groupName": "containerRepository", "properties": { "EditableOptions": "True" } @@ -76,36 +63,35 @@ "rows": "2" }, "label": "ms-resource:loc.input.label.tags", - "groupName": "commands", "helpMarkDown": "ms-resource:loc.input.help.tags" }, { "name": "arguments", "type": "string", "label": "ms-resource:loc.input.label.arguments", - "groupName": "commands", "helpMarkDown": "ms-resource:loc.input.help.arguments" }, { - "name": "addPipelineData", + "name": "addPipelineMetadata", "type": "boolean", - "label": "ms-resource:loc.input.label.addPipelineData", - "groupName": "commands", + "label": "ms-resource:loc.input.label.addPipelineMetadata", "defaultValue": "true", - "helpMarkDown": "ms-resource:loc.input.help.addPipelineData" + "helpMarkDown": "ms-resource:loc.input.help.addPipelineMetadata" } ], "instanceNameFormat": "ms-resource:loc.instanceNameFormat", "execution": { "Node": { - "target": "src//buildctl.js" + "target": "src//buildcontainer.js" } }, "messages": { - "NotAValidSemverVersion": "ms-resource:loc.messages.NotAValidSemverVersion", - "VerifyBuildctlInstallation": "ms-resource:loc.messages.VerifyBuildctlInstallation", + "ContainerPatternNotFound": "ms-resource:loc.messages.ContainerPatternNotFound", "BuildctlLatestNotKnown": "ms-resource:loc.messages.BuildctlLatestNotKnown", "BuildctlDownloadFailed": "ms-resource:loc.messages.BuildctlDownloadFailed", - "BuildctlNotFoundInFolder": "ms-resource:loc.messages.BuildctlNotFoundInFolder" + "BuildctlNotFoundInFolder": "ms-resource:loc.messages.BuildctlNotFoundInFolder", + "FileContentSynced": "ms-resource:loc.messages.FileContentSynced", + "VerifyBuildctlInstallation": "ms-resource:loc.messages.VerifyBuildctlInstallation", + "WritingDockerConfigToTempFile": "ms-resource:loc.messages.WritingDockerConfigToTempFile" } } \ No newline at end of file