Skip to content

Commit

Permalink
changes to remove metadata
Browse files Browse the repository at this point in the history
  • Loading branch information
@prebansa @leantk
prebansa authored and leantk committed Dec 23, 2019
1 parent 358a2bc commit 1ae43e3
Show file tree
Hide file tree
Showing 8 changed files with 67 additions and 203 deletions.
7 changes: 3 additions & 4 deletions Tasks/ContainerBuildV0/Strings/resources.resjson/en-US/resources.resjson
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
"loc.helpMarkDown": "[Learn more about this task](https://go.microsoft.com/fwlink/?linkid=2107300)",
"loc.description": "Build Task",
"loc.instanceNameFormat": "Container Build Task",
"loc.input.label.poolService": "Pool Service (applicable when running on kubernetes)",
"loc.input.help.poolService": "Name of the service applicable when running on kubernetes.",
"loc.input.label.dockerRegistryServiceConnection": "Docker registry service connection",
"loc.input.help.dockerRegistryServiceConnection": "Select a Docker registry service connection.",
"loc.input.label.repository": "Container repository",
Expand All @@ -13,11 +15,8 @@
"loc.input.help.buildContext": "Path to Build context.",
"loc.input.label.tags": "Tags",
"loc.input.help.tags": "A list of tags in separate lines. Tags are used while building and pushing the image to container registry.",
"loc.input.label.arguments": "Arguments",
"loc.input.help.arguments": "Arguments used while building the image.",
"loc.input.label.addPipelineMetadata": "Add Pipeline metadata to image(s)",
"loc.input.help.addPipelineMetadata": "By default pipeline data like source branch name, build id are added which helps with traceability. For example you can inspect an image to find out which pipeline built the image. You can opt out of this default behavior by using this input.",
"loc.messages.ContainerPatternNotFound": "No pattern found in Docker filepath parameter",
"loc.messages.CouldNotFindDockerConfig": "Could not find Docker Config. Either DOCKER_CONFIG variable is not set, or the config file is outside the temp directory, or the file does not exist. DOCKER_CONFIG: %s",
"loc.messages.BuildctlLatestNotKnown": "Cannot get the latest Buildctl info from %s. Error %s. Using default Buildctl version %s.",
"loc.messages.BuildctlDownloadFailed": "Failed to download Buildctl from location %s. Error %s",
"loc.messages.BuildctlNotFoundInFolder": "Buildctl executable not found in path %s",
Expand Down
20 changes: 14 additions & 6 deletions Tasks/ContainerBuildV0/src/buildctl.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import ContainerConnection from "docker-common-v2/containerconnection";
import { getDockerRegistryEndpointAuthenticationToken } from "docker-common-v2/registryauthenticationprovider/registryauthenticationtoken";

async function configureBuildctl() {

tl.debug("configuring buildctl");
var stableBuildKitVersion = await utils.getStableBuildctlVersion();
var buildctlPath = await utils.downloadBuildctl(stableBuildKitVersion);

Expand All @@ -19,10 +21,11 @@ async function configureBuildctl() {
}

async function verifyBuildctl() {
await configureBuildctl();
var buildctlToolPath = tl.which("buildctl", true);
if(buildctlToolPath == "")
await configureBuildctl();
tl.debug(tl.loc("VerifyBuildctlInstallation"));

var buildctlToolPath = tl.which("buildctl", true);
var buildctlTool = tl.tool(buildctlToolPath);

buildctlTool.arg("--help");
Expand Down Expand Up @@ -54,9 +57,6 @@ export async function buildctlBuildAndPush() {
imageNames.push(imageName);
}
}
else {
imageNames = connection.getQualifiedImageNamesFromConfig(repositoryName, true);
}

var contextarg = "--local=context="+tl.getInput("buildContext", true);
var dockerfilearg = "--local=dockerfile="+tl.getInput("Dockerfile", true);
Expand All @@ -72,9 +72,17 @@ export async function buildctlBuildAndPush() {
if (tags && tags.length > 0) {
tags.forEach(async tag => {
buildctlTool.arg(`--output=type=image,name=${imageName}:${tag},push=true`);
await buildctlTool.exec();
buildctlTool.exec();
})
}
else {
buildctlTool.arg(`--output=type=image,name=${imageName}:latest,push=true`);
buildctlTool.exec();
}
})
}
else {
// only build the image
await buildctlTool.exec();
}
}
14 changes: 6 additions & 8 deletions Tasks/ContainerBuildV0/src/docker.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,6 @@ export async function dockerBuildAndPush() {
let endpointId = tl.getInput("dockerRegistryServiceConnection");
let registryAuthenticationToken: RegistryAuthenticationToken = getDockerRegistryEndpointAuthenticationToken(endpointId);

// Take the specified command
/*let command = tl.getInput("command", true).toLowerCase();
let isLogout = (command === "logout");*/

// Connect to any specified container registry
let connection = new ContainerConnection();
connection.open(null, registryAuthenticationToken, true, false);
Expand All @@ -25,8 +21,10 @@ export async function dockerBuildAndPush() {
resultPaths += pathToResult;
})

commandImplementation = require("./dockerpush")
await commandImplementation.run(connection, (pathToResult) => {
resultPaths += pathToResult;
})
if (tl.getInput("dockerRegistryServiceConnection")) {
commandImplementation = require("./dockerpush")
await commandImplementation.run(connection, (pathToResult) => {
resultPaths += pathToResult;
})
}
}
3 changes: 1 addition & 2 deletions Tasks/ContainerBuildV0/src/dockerbuild.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,8 @@ export async function runBuild(connection: ContainerConnection, outputUpdate: (d
imageNames = connection.getQualifiedImageNamesFromConfig(repositoryName, true);
}

const addPipelineData = tl.getBoolInput("addPipelineMetadata");
// get label arguments
let labelArguments = pipelineUtils.getDefaultLabels(addPipelineData);
let labelArguments: string[] = [];

// get tags input
let tags = tl.getDelimitedInput("tags", "\n");
Expand Down
149 changes: 1 addition & 148 deletions Tasks/ContainerBuildV0/src/dockerpush.ts
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,13 @@
"use strict";

import * as tl from "azure-pipelines-task-lib/task";
import * as fs from 'fs';
import ContainerConnection from "docker-common-v2/containerconnection";
import * as dockerCommandUtils from "docker-common-v2/dockercommandutils";
import * as utils from "./utils";
import { findDockerFile } from "docker-common-v2/fileutils";
import { WebRequest, WebResponse, sendRequest } from "azure-arm-rest-v2/webClient";
import { getBaseImageName, getResourceName, getBaseImageNameFromDockerFile } from "docker-common-v2/containerimageutils";
import * as pipelineUtils from "docker-common-v2/pipelineutils";

import Q = require('q');

const matchPatternForDigestAndSize = new RegExp(/sha256\:([\w]+)(\s+)size\:\s([\w]+)/);

function pushMultipleImages(connection: ContainerConnection, imageNames: string[], tags: string[], commandArguments: string, onCommandOut: (image, output) => any): any {
let promise: Q.Promise<void>;
// create chained promise of push commands
Expand Down Expand Up @@ -92,18 +86,9 @@ export async function run(connection: ContainerConnection, outputUpdate: (data:
// push all tags
let output = "";
let outputImageName = "";
let digest = "";
let imageSize = "";
let promise = pushMultipleImages(connection, imageNames, tags, commandArguments, (image, commandOutput) => {
output += commandOutput;
outputImageName = image;
let digest = extractDigestFromOutput(commandOutput, matchPatternForDigestAndSize);
tl.debug("outputImageName: " + outputImageName + "\n" + "commandOutput: " + commandOutput + "\n" + "digest:" + digest + "imageSize:" + imageSize);
publishToImageMetadataStore(connection, outputImageName, tags, digest, dockerFile).then((result) => {
tl.debug("ImageDetailsApiResponse: " + JSON.stringify(result));
}, (error) => {
tl.warning("publishToImageMetadataStore failed with error: " + error);
});
});

if (promise) {
Expand All @@ -118,136 +103,4 @@ export async function run(connection: ContainerConnection, outputUpdate: (data:
}

return promise;
}

async function publishToImageMetadataStore(connection: ContainerConnection, imageName: string, tags: string[], digest: string, dockerFilePath: string): Promise<any> {
// Getting imageDetails
const imageUri = getResourceName(imageName, digest);
const baseImageName = dockerFilePath ? getBaseImageNameFromDockerFile(dockerFilePath) : "NA";
const history = await dockerCommandUtils.getHistory(connection, imageName);
if (!history) {
return null;
}

const layers = dockerCommandUtils.getLayers(history);
const imageSize = dockerCommandUtils.getImageSize(layers);

// Get data for ImageFingerPrint
// v1Name is the layerID for the final layer in the image
// v2Blobs are ordered list of layers that represent the given image, obtained from docker inspect output
const v1Name = dockerCommandUtils.getImageFingerPrintV1Name(history);
const imageRootfsLayers = await dockerCommandUtils.getImageRootfsLayers(connection, v1Name);
let imageFingerPrint: { [key: string]: string | string[] } = {};
if (imageRootfsLayers && imageRootfsLayers.length > 0) {
imageFingerPrint = dockerCommandUtils.getImageFingerPrint(imageRootfsLayers, v1Name);
}

const addPipelineData = tl.getBoolInput("addPipelineMetadata");

// Getting pipeline variables
const build = "build";
const hostType = tl.getVariable("System.HostType").toLowerCase();
const runId = hostType === build ? parseInt(tl.getVariable("Build.BuildId")) : parseInt(tl.getVariable("Release.ReleaseId"));
const pipelineVersion = addPipelineData ? hostType === build ? tl.getVariable("Build.BuildNumber") : tl.getVariable("Release.ReleaseName") : "";
const pipelineName = addPipelineData ? tl.getVariable("System.DefinitionName") : "";
const pipelineId = addPipelineData ? tl.getVariable("System.DefinitionId") : "";
const jobName = addPipelineData ? tl.getVariable("System.PhaseDisplayName") : "";
const creator = addPipelineData ? dockerCommandUtils.getCreatorEmail() : "";
const logsUri = addPipelineData ? dockerCommandUtils.getPipelineLogsUrl() : "";
const artifactStorageSourceUri = addPipelineData ? dockerCommandUtils.getPipelineUrl() : "";

const repoUrl = tl.getVariable("Build.Repository.Uri");
const contextUrl = addPipelineData && repoUrl ? repoUrl : "";

const commitId = tl.getVariable("Build.SourceVersion");
const revisionId = addPipelineData && commitId ? commitId : "";

const labelArguments = pipelineUtils.getDefaultLabels(addPipelineData);
const buildOptions = dockerCommandUtils.getBuildAndPushArguments(dockerFilePath, labelArguments, tags);

// Capture Repository data for Artifact traceability
const repositoryTypeName = tl.getVariable("Build.Repository.Provider");
const repositoryId = tl.getVariable("Build.Repository.ID");
const repositoryName = tl.getVariable("Build.Repository.Name");
const branch = tl.getVariable("Build.SourceBranchName");

const requestUrl = tl.getVariable("System.TeamFoundationCollectionUri") + tl.getVariable("System.TeamProject") + "/_apis/deployment/imagedetails?api-version=5.0-preview.1";
const requestBody: string = JSON.stringify(
{
"imageName": imageUri,
"imageUri": imageUri,
"hash": digest,
"baseImageName": baseImageName,
"distance": layers.length,
"imageType": "",
"mediaType": "",
"tags": tags,
"layerInfo": layers,
"runId": runId,
"pipelineVersion": pipelineVersion,
"pipelineName": pipelineName,
"pipelineId": pipelineId,
"jobName": jobName,
"imageSize": imageSize,
"creator": creator,
"logsUri": logsUri,
"artifactStorageSourceUri": artifactStorageSourceUri,
"contextUrl": contextUrl,
"revisionId": revisionId,
"buildOptions": buildOptions,
"repositoryTypeName": repositoryTypeName,
"repositoryId": repositoryId,
"repositoryName": repositoryName,
"branch": branch,
"imageFingerPrint": imageFingerPrint
}
);

return sendRequestToImageStore(requestBody, requestUrl);
}

function extractDigestFromOutput(dockerPushCommandOutput: string, matchPattern: RegExp): string {
// SampleCommandOutput : The push refers to repository [xyz.azurecr.io/acr-helloworld]
// 3b7670606102: Pushed
// e2af85e4b310: Pushed ce8609e9fdad: Layer already exists
// f2b18e6d6636: Layer already exists
// 62: digest: sha256:5e3c9cf1692e129744fe7db8315f05485c6bb2f3b9f6c5096ebaae5d5bfbbe60 size: 5718

// Below regex will extract part after sha256, so expected return value will be 5e3c9cf1692e129744fe7db8315f05485c6bb2f3b9f6c5096ebaae5d5bfbbe60
const imageMatch = dockerPushCommandOutput.match(matchPattern);
let digest = "";
if (imageMatch && imageMatch.length >= 1) {
digest = imageMatch[1];
}

return digest;
}

async function sendRequestToImageStore(requestBody: string, requestUrl: string): Promise<any> {
const request = new WebRequest();
const accessToken: string = tl.getEndpointAuthorizationParameter('SYSTEMVSSCONNECTION', 'ACCESSTOKEN', false);
request.uri = requestUrl;
request.method = 'POST';
request.body = requestBody;
request.headers = {
"Content-Type": "application/json",
"Authorization": "Bearer " + accessToken
};

tl.debug("requestUrl: " + requestUrl);
tl.debug("requestBody: " + requestBody);
tl.debug("accessToken: " + accessToken);

try {
tl.debug("Sending request for pushing image to Image meta data store");
const response = await sendRequest(request);
return response;
}
catch (error) {
tl.debug("Unable to push to Image Details Artifact Store, Error: " + error);
}

return Promise.resolve();
}


}
29 changes: 20 additions & 9 deletions Tasks/ContainerBuildV0/src/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@ import * as fileutils from "docker-common-v2/fileutils";
const buildctlToolName = "buildctl"
const uuidV4 = require('uuid/v4');
const buildctlLatestReleaseUrl = "https://api.github.com/repos/moby/buildkit/releases/latest";
const buildctlToolNameWithExtension = buildctlToolName + getExecutableExtension();
const stableBuildctlVersion = "v0.5.1"
const serviceidentifier = "k8loadbalancer"
var serviceName = "k8s-poolprovider"
var namespace = "azuredevops"
var port = "8082"
var clusteruri = ""
Expand Down Expand Up @@ -46,11 +47,11 @@ export async function downloadBuildctl(version: string): Promise<string> {
}

var unzipedBuildctlPath = await toolLib.extractTar(buildctlDownloadPath);
unzipedBuildctlPath = path.join(unzipedBuildctlPath, "bin", buildctlToolName);
unzipedBuildctlPath = path.join(unzipedBuildctlPath, "bin", buildctlToolNameWithExtension);

tl.debug('Extracting archive: ' + unzipedBuildctlPath+' download path: '+buildctlDownloadPath);

var cachedToolpath = await toolLib.cacheFile(unzipedBuildctlPath, buildctlToolName, buildctlToolName, version);
var cachedToolpath = await toolLib.cacheFile(unzipedBuildctlPath, buildctlToolNameWithExtension, buildctlToolName, version);

tl.debug('CachedTool path: ' + cachedToolpath);
}
Expand Down Expand Up @@ -85,17 +86,20 @@ export async function getServiceDetails() {

var kubectlToolPath = tl.which("kubectl", true);
var kubectlTool = tl.tool(kubectlToolPath);

var serviceNameInput = tl.getInput('poolService', false);
if (serviceNameInput != null) {
serviceName = serviceNameInput;
}
kubectlTool.arg('get');
kubectlTool.arg('service');
kubectlTool.arg(`--selector=identifier=${serviceidentifier}`);
kubectlTool.arg(`${serviceName}`);
kubectlTool.arg('-o=json');

var serviceResponse= kubectlTool.execSync();

namespace = JSON.parse(serviceResponse.stdout).items[0].metadata.namespace;
port = JSON.parse(serviceResponse.stdout).items[0].spec.ports[0].port;
clusteruri = JSON.parse(serviceResponse.stdout).items[0].status.loadBalancer.ingress[0].ip;
namespace = JSON.parse(serviceResponse.stdout).metadata.namespace;
port = JSON.parse(serviceResponse.stdout).spec.ports[0].port;
clusteruri = JSON.parse(serviceResponse.stdout).status.loadBalancer.ingress[0].ip;
}

export async function getBuildKitPod() {
Expand Down Expand Up @@ -123,7 +127,7 @@ export async function getBuildKitPod() {

function findBuildctl(rootFolder: string) {

var BuildctlPath = path.join(rootFolder, buildctlToolName);
var BuildctlPath = path.join(rootFolder, buildctlToolNameWithExtension);
var allPaths = tl.find(rootFolder);
var matchingResultsFiles = tl.match(allPaths, BuildctlPath, rootFolder);

Expand All @@ -139,6 +143,13 @@ function getArchiveExtension(): string {
return ".tar.gz";
}

function getExecutableExtension(): string {
if (os.type() == 'Windows_NT') {
return ".exe";
}
return "";
}

function getTaskOutputDir(command: string): string {
let tempDirectory = tl.getVariable('agent.tempDirectory') || os.tmpdir();
let taskOutputDir = path.join(tempDirectory, "task_outputs");
Expand Down
Loading

0 comments on commit 1ae43e3

Please sign in to comment.