diff --git a/Tasks/DownloadBuildArtifactsV0/Strings/resources.resjson/en-US/resources.resjson b/Tasks/DownloadBuildArtifactsV0/Strings/resources.resjson/en-US/resources.resjson index 15b9ce747176..643c2a5b2061 100644 --- a/Tasks/DownloadBuildArtifactsV0/Strings/resources.resjson/en-US/resources.resjson +++ b/Tasks/DownloadBuildArtifactsV0/Strings/resources.resjson/en-US/resources.resjson @@ -34,6 +34,7 @@ "loc.messages.DownloadArtifacts": "Downloading artifact %s from: %s", "loc.messages.DownloadingArtifactsForBuild": "Downloading artifacts for build: %s", "loc.messages.LinkedArtifactCount": "Linked artifacts count: %s", + "loc.messages.ExtractionFailed": "Failed to extract package with error %s", "loc.messages.FileContainerInvalidArtifactData": "Invalid file container artifact. Resource data must be in the format #/{container id}/path", "loc.messages.UnsupportedArtifactType": "Unsupported artifact type: %s", "loc.messages.BuildNotFound": "Build with ID %s not found", diff --git a/Tasks/DownloadBuildArtifactsV0/main.ts b/Tasks/DownloadBuildArtifactsV0/main.ts index 78ae428f21d3..4c8c68bb80d5 100644 --- a/Tasks/DownloadBuildArtifactsV0/main.ts +++ b/Tasks/DownloadBuildArtifactsV0/main.ts @@ -13,6 +13,8 @@ import * as engine from 'artifact-engine/Engine'; import * as providers from 'artifact-engine/Providers'; import * as webHandlers from 'artifact-engine/Providers/typed-rest-client/Handlers'; +var DecompressZip = require('decompress-zip'); + var taskJson = require('./task.json'); tl.setResourcePath(path.join(__dirname, 'task.json')); @@ -82,15 +84,11 @@ async function main(): Promise { var accessToken: string = tl.getEndpointAuthorizationParameter('SYSTEMVSSCONNECTION', 'AccessToken', false); var credentialHandler: IRequestHandler = getHandlerFromToken(accessToken); var webApi: WebApi = new WebApi(endpointUrl, credentialHandler); - var debugMode: string = tl.getVariable('System.Debug'); - var isVerbose: boolean = debugMode ? debugMode.toLowerCase() != 'false' : false; - var parallelLimit: number = +tl.getInput("parallelizationLimit", false); var retryLimit = parseInt(tl.getVariable("VSTS_HTTP_RETRY")) ? parseInt(tl.getVariable("VSTS_HTTP_RETRY")) : 4; var templatePath: string = path.join(__dirname, 'vsts.handlebars.txt'); var buildApi: IBuildApi = await webApi.getBuildApi(); var artifacts = []; - var itemPattern: string = tl.getInput("itemPattern", false) || '**'; if (isCurrentBuild) { projectId = tl.getVariable("System.TeamProjectId"); @@ -248,44 +246,62 @@ async function main(): Promise { if (artifacts) { var downloadPromises: Array> = []; artifacts.forEach(async function (artifact, index, artifacts) { - let downloaderOptions = new engine.ArtifactEngineOptions(); - downloaderOptions.itemPattern = itemPattern; - downloaderOptions.verbose = isVerbose; - - if (parallelLimit) { - downloaderOptions.parallelProcessingLimit = parallelLimit; - } + let downloaderOptions = configureDownloaderOptions(); if (artifact.resource.type.toLowerCase() === "container") { - let downloader = new engine.ArtifactEngine(); + var handler = new webHandlers.PersonalAccessTokenCredentialHandler(accessToken); + var isPullRequestFork = tl.getVariable("SYSTEM.PULLREQUEST.ISFORK"); + var isPullRequestForkBool = isPullRequestFork ? isPullRequestFork.toLowerCase() == 'true' : false; - console.log(tl.loc("DownloadingContainerResource", artifact.resource.data)); - var containerParts = artifact.resource.data.split('/'); + if (isPullRequestForkBool) { + const archiveUrl: string = endpointUrl + "/" + projectId + "/_apis/build/builds/" + buildId + "/artifacts?artifactName=" + artifact.name + "&$format=zip"; + console.log(tl.loc("DownloadArtifacts", artifact.name, archiveUrl)); - if (containerParts.length < 3) { - throw new Error(tl.loc("FileContainerInvalidArtifactData")); - } - - var containerId = parseInt(containerParts[1]); - var containerPath = containerParts.slice(2,containerParts.length).join('/'); - - if (containerPath == "/") { - //container REST api oddity. Passing '/' as itemPath downloads the first file instead of returning the meta data about the all the files in the root level. - //This happens only if the first item is a file. - containerPath = "" - } + var zipLocation = path.join(downloadPath, artifact.name + ".zip"); + await getZipFromUrl(archiveUrl, zipLocation, handler, downloaderOptions); - var itemsUrl = endpointUrl + "/_apis/resources/Containers/" + containerId + "?itemPath=" + encodeURIComponent(containerPath) + "&isShallow=true&api-version=4.1-preview.4"; - console.log(tl.loc("DownloadArtifacts", artifact.name, itemsUrl)); + var unzipPromise = unzip(zipLocation, downloadPath); + unzipPromise.catch((error) => { + throw error; + }); - var variables = {}; - var handler = new webHandlers.PersonalAccessTokenCredentialHandler(accessToken); - var webProvider = new providers.WebProvider(itemsUrl, templatePath, variables, handler); - var fileSystemProvider = new providers.FilesystemProvider(downloadPath); + downloadPromises.push(unzipPromise); + await unzipPromise; - downloadPromises.push(downloader.processItems(webProvider, fileSystemProvider, downloaderOptions).catch((reason) => { - reject(reason); - })); + if (tl.exist(zipLocation)) { + tl.rmRF(zipLocation); + } + } + else { + let downloader = new engine.ArtifactEngine(); + + console.log(tl.loc("DownloadingContainerResource", artifact.resource.data)); + var containerParts = artifact.resource.data.split('/'); + + if (containerParts.length < 3) { + throw new Error(tl.loc("FileContainerInvalidArtifactData")); + } + + var containerId = parseInt(containerParts[1]); + var containerPath = containerParts.slice(2,containerParts.length).join('/'); + + if (containerPath == "/") { + //container REST api oddity. Passing '/' as itemPath downloads the first file instead of returning the meta data about the all the files in the root level. + //This happens only if the first item is a file. + containerPath = "" + } + + var itemsUrl = endpointUrl + "/_apis/resources/Containers/" + containerId + "?itemPath=" + encodeURIComponent(containerPath) + "&isShallow=true&api-version=4.1-preview.4"; + console.log(tl.loc("DownloadArtifacts", artifact.name, itemsUrl)); + + var variables = {}; + var webProvider = new providers.WebProvider(itemsUrl, templatePath, variables, handler); + var fileSystemProvider = new providers.FilesystemProvider(downloadPath); + + downloadPromises.push(downloader.processItems(webProvider, fileSystemProvider, downloaderOptions).catch((reason) => { + reject(reason); + })); + } } else if (artifact.resource.type.toLowerCase() === "filepath") { let downloader = new engine.ArtifactEngine(); @@ -345,6 +361,42 @@ function executeWithRetriesImplementation(operationName: string, operation: () = }); } +async function getZipFromUrl(artifactArchiveUrl: string, localPathRoot: string, handler: webHandlers.PersonalAccessTokenCredentialHandler, downloaderOptions: engine.ArtifactEngineOptions) { + var downloader = new engine.ArtifactEngine(); + var zipProvider = new providers.ZipProvider(artifactArchiveUrl, handler); + var filesystemProvider = new providers.FilesystemProvider(localPathRoot); + + await downloader.processItems(zipProvider, filesystemProvider, downloaderOptions) +} + +function configureDownloaderOptions(): engine.ArtifactEngineOptions { + var downloaderOptions = new engine.ArtifactEngineOptions(); + downloaderOptions.itemPattern = tl.getInput('itemPattern', false) || "**"; + downloaderOptions.parallelProcessingLimit = +tl.getVariable("release.artifact.download.parallellimit") || 8; + var debugMode = tl.getVariable('System.Debug'); + downloaderOptions.verbose = debugMode ? debugMode.toLowerCase() != 'false' : false; + + return downloaderOptions; +} + +export async function unzip(zipLocation: string, unzipLocation: string): Promise { + await new Promise(function (resolve, reject) { + tl.debug('Extracting ' + zipLocation + ' to ' + unzipLocation); + + var unzipper = new DecompressZip(zipLocation); + unzipper.on('error', err => { + return reject(tl.loc("ExtractionFailed", err)) + }); + unzipper.on('extract', log => { + tl.debug('Extracted ' + zipLocation + ' to ' + unzipLocation + ' successfully'); + return resolve(); + }); + unzipper.extract({ + path: unzipLocation + }); + }); +} + main() .then((result) => tl.setResult(tl.TaskResult.Succeeded, "")) .catch((err) => { diff --git a/Tasks/DownloadBuildArtifactsV0/package-lock.json b/Tasks/DownloadBuildArtifactsV0/package-lock.json index efdb8d7d8e66..bfa04f5eda38 100644 --- a/Tasks/DownloadBuildArtifactsV0/package-lock.json +++ b/Tasks/DownloadBuildArtifactsV0/package-lock.json @@ -9,6 +9,11 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-6.14.3.tgz", "integrity": "sha512-V2VrQBCKo4U0rni6tW4AASRDqIO5ZTLDN/Xzrm4mNBr9SGQYZ+7zZJn+hMs89Q8ZCIHzp4aWQPyCpK+rux1YGA==" }, + "abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" + }, "align-text": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/align-text/-/align-text-0.1.4.tgz", @@ -56,25 +61,14 @@ "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=" }, "azure-devops-node-api": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/azure-devops-node-api/-/azure-devops-node-api-7.0.0.tgz", - "integrity": "sha512-WXTqFDE2QhfKli1EkcaMbGYuDpOVcNoccnQBF/bZCkPZsogLJOnsZHO/BJnd2VrT+eSJtPoVcHWjKfE/Zcihew==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/azure-devops-node-api/-/azure-devops-node-api-7.2.0.tgz", + "integrity": "sha512-pMfGJ6gAQ7LRKTHgiRF+8iaUUeGAI0c8puLaqHLc7B8AR7W6GJLozK9RFeUHFjEGybC9/EB3r67WPd7e46zQ8w==", "requires": { "os": "0.1.1", "tunnel": "0.0.4", - "typed-rest-client": "1.0.9", + "typed-rest-client": "1.2.0", "underscore": "1.8.3" - }, - "dependencies": { - "typed-rest-client": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/typed-rest-client/-/typed-rest-client-1.0.9.tgz", - "integrity": "sha512-iOdwgmnP/tF6Qs+oY4iEtCf/3fnCDl7Gy9LGPJ4E3M4Wj3uaSko15FVwbsaBmnBqTJORnXBWVY5306D4HH8oiA==", - "requires": { - "tunnel": "0.0.4", - "underscore": "1.8.3" - } - } } }, "balanced-match": { @@ -82,6 +76,15 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, + "binary": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", + "integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=", + "requires": { + "buffers": "0.1.1", + "chainsaw": "0.1.0" + } + }, "brace-expansion": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.8.tgz", @@ -91,6 +94,11 @@ "concat-map": "0.0.1" } }, + "buffers": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", + "integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s=" + }, "camelcase": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-1.2.1.tgz", @@ -107,6 +115,14 @@ "lazy-cache": "1.0.4" } }, + "chainsaw": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", + "integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=", + "requires": { + "traverse": "0.3.9" + } + }, "cliui": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/cliui/-/cliui-2.1.0.tgz", @@ -131,12 +147,36 @@ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, "decamelize": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", "optional": true }, + "decompress-zip": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/decompress-zip/-/decompress-zip-0.3.0.tgz", + "integrity": "sha1-rjvLfjTGWHmt/nfhnDD4ZgK0vbA=", + "requires": { + "binary": "0.3.0", + "graceful-fs": "4.1.15", + "mkpath": "0.1.0", + "nopt": "3.0.6", + "q": "1.5.1", + "readable-stream": "1.1.14", + "touch": "0.0.3" + } + }, + "graceful-fs": { + "version": "4.1.15", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz", + "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==" + }, "handlebars": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.0.10.tgz", @@ -148,11 +188,21 @@ "uglify-js": "2.8.29" } }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, "is-buffer": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" }, + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, "kind-of": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", @@ -185,11 +235,24 @@ "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=" }, + "mkpath": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/mkpath/-/mkpath-0.1.0.tgz", + "integrity": "sha1-dVSm+Nhxg0zJe1RisSLEwSTW3pE=" + }, "mockery": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/mockery/-/mockery-1.7.0.tgz", "integrity": "sha1-9O3g2HUMHJcnwnLqLGBiniyaHE8=" }, + "nopt": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", + "integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=", + "requires": { + "abbrev": "1.1.1" + } + }, "optimist": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", @@ -209,6 +272,17 @@ "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=" }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "0.0.1", + "string_decoder": "0.10.31" + } + }, "repeat-string": { "version": "1.6.1", "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", @@ -241,11 +315,48 @@ "amdefine": "1.0.1" } }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + }, + "touch": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/touch/-/touch-0.0.3.tgz", + "integrity": "sha1-Ua7z1ElXHU8oel2Hyci0kYGg2x0=", + "requires": { + "nopt": "1.0.10" + }, + "dependencies": { + "nopt": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz", + "integrity": "sha1-bd0hvSoxQXuScn3Vhfim83YI6+4=", + "requires": { + "abbrev": "1.1.1" + } + } + } + }, + "traverse": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", + "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=" + }, "tunnel": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.4.tgz", "integrity": "sha1-LTeFoVjBdMmhbcLARuxfxfF0IhM=" }, + "typed-rest-client": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/typed-rest-client/-/typed-rest-client-1.2.0.tgz", + "integrity": "sha512-FrUshzZ1yxH8YwGR29PWWnfksLEILbWJydU7zfIRkyH7kAEzB62uMAl2WY6EyolWpLpVHeJGgQm45/MaruaHpw==", + "requires": { + "tunnel": "0.0.4", + "underscore": "1.8.3" + } + }, "uglify-js": { "version": "2.8.29", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-2.8.29.tgz", diff --git a/Tasks/DownloadBuildArtifactsV0/package.json b/Tasks/DownloadBuildArtifactsV0/package.json index 3c02f89593a4..24438c8b1235 100644 --- a/Tasks/DownloadBuildArtifactsV0/package.json +++ b/Tasks/DownloadBuildArtifactsV0/package.json @@ -20,6 +20,7 @@ "vsts-task-lib": "2.1.0", "azure-devops-node-api": "7.2.0", "artifact-engine": "0.1.26", - "@types/node": "^6.0.101" + "@types/node": "^6.0.101", + "decompress-zip": "0.3.0" } } diff --git a/Tasks/DownloadBuildArtifactsV0/task.json b/Tasks/DownloadBuildArtifactsV0/task.json index 935ea4349ebd..4e45c2a64986 100644 --- a/Tasks/DownloadBuildArtifactsV0/task.json +++ b/Tasks/DownloadBuildArtifactsV0/task.json @@ -10,7 +10,7 @@ "version": { "Major": 0, "Minor": 152, - "Patch": 0 + "Patch": 1 }, "groups": [ { @@ -239,6 +239,7 @@ "DownloadArtifacts": "Downloading artifact %s from: %s", "DownloadingArtifactsForBuild": "Downloading artifacts for build: %s", "LinkedArtifactCount": "Linked artifacts count: %s", + "ExtractionFailed": "Failed to extract package with error %s", "FileContainerInvalidArtifactData": "Invalid file container artifact. Resource data must be in the format #/{container id}/path", "UnsupportedArtifactType": "Unsupported artifact type: %s", "BuildNotFound": "Build with ID %s not found", diff --git a/Tasks/DownloadBuildArtifactsV0/task.loc.json b/Tasks/DownloadBuildArtifactsV0/task.loc.json index 68a0a189e922..887aedf0a86d 100644 --- a/Tasks/DownloadBuildArtifactsV0/task.loc.json +++ b/Tasks/DownloadBuildArtifactsV0/task.loc.json @@ -10,7 +10,7 @@ "version": { "Major": 0, "Minor": 152, - "Patch": 0 + "Patch": 1 }, "groups": [ { @@ -239,6 +239,7 @@ "DownloadArtifacts": "ms-resource:loc.messages.DownloadArtifacts", "DownloadingArtifactsForBuild": "ms-resource:loc.messages.DownloadingArtifactsForBuild", "LinkedArtifactCount": "ms-resource:loc.messages.LinkedArtifactCount", + "ExtractionFailed": "ms-resource:loc.messages.ExtractionFailed", "FileContainerInvalidArtifactData": "ms-resource:loc.messages.FileContainerInvalidArtifactData", "UnsupportedArtifactType": "ms-resource:loc.messages.UnsupportedArtifactType", "BuildNotFound": "ms-resource:loc.messages.BuildNotFound",