From 341bfb0bfcf473b88205f486c861ff543bf4abd1 Mon Sep 17 00:00:00 2001 From: Aasim Malladi Date: Tue, 23 Jul 2019 13:06:14 -0700 Subject: [PATCH 1/2] Updated download task to support package names --- Tasks/DownloadPackageV1/main.ts | 8 ++++++ Tasks/DownloadPackageV1/package.ts | 36 +++++++++++++++++++++++++-- Tasks/DownloadPackageV1/task.json | 9 ++++--- Tasks/DownloadPackageV1/task.loc.json | 9 ++++--- 4 files changed, 54 insertions(+), 8 deletions(-) diff --git a/Tasks/DownloadPackageV1/main.ts b/Tasks/DownloadPackageV1/main.ts index 13b362a2dd77..7eee2cabfda6 100644 --- a/Tasks/DownloadPackageV1/main.ts +++ b/Tasks/DownloadPackageV1/main.ts @@ -61,6 +61,14 @@ async function main(): Promise { .matchingPattern(files) .withRetries(Retry(retryLimit)) .build(); + + const regexGuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + + if(!regexGuid.test(packageId)){ + tl.debug("Trying to resolve package name " + packageId + " to id."); + packageId = await p.resolvePackageId(feed.feedId, feed.projectId, packageId); + tl.debug("Resolved package id: " + packageId); + } const packageFiles: PackageFile[] = await p.download(feed.feedId, feed.projectId, packageId, version, downloadPath, extractPackage); diff --git a/Tasks/DownloadPackageV1/package.ts b/Tasks/DownloadPackageV1/package.ts index d7854867375b..8a6cf92433f1 100644 --- a/Tasks/DownloadPackageV1/package.ts +++ b/Tasks/DownloadPackageV1/package.ts @@ -36,6 +36,7 @@ export abstract class Package { private executeWithRetries: (operation: () => Promise) => Promise; private packagingAreaName: string = "Packaging"; + private getPackagesAreaId: string = "7a20d846-c929-4acc-9ea2-0d5a7df1b197"; private packagingMetadataAreaId: string; constructor(builder: PackageUrlsBuilder) { @@ -78,11 +79,11 @@ export abstract class Package { }); } - protected async getPackageMetadata(connection: WebApi, routeValues: any, queryParams?: any): Promise { + protected async getPackageMetadata(connection: WebApi, routeValues: any, queryParams?: any, areaId?: string): Promise { var metadataUrl = await this.getUrl( connection.vsoClient, this.packagingAreaName, - this.packagingMetadataAreaId, + areaId || this.packagingMetadataAreaId, routeValues, queryParams ); @@ -105,6 +106,37 @@ export abstract class Package { }); } + public async resolvePackageId( + feedId: string, + project: string, + packageName: string + ): Promise { + const routeValues = { + feedId: feedId, + project: project + }; + const queryParams = { + packageNameQuery: packageName, + protocolType: this.packageProtocolAreaName + }; + + return new Promise(async (resolve, reject) => { + this.getPackageMetadata(this.feedConnection, routeValues, queryParams, this.getPackagesAreaId) + .then(packages => { + tl.debug("Found " + packages["count"] + " packages matching search pattern " + packageName); + for (let i = 0; i < packages["count"]; i++) { + if (packages["value"][i]["name"] == packageName) { + return resolve(packages["value"][i]["id"]); + } + } + return reject("Package with name " + packageName + " not found."); + }).catch(error => { + tl.debug("Package with name " + packageName + " not found: " + error); + return reject(error); + }) + }); + } + public async download( feedId: string, project: string, diff --git a/Tasks/DownloadPackageV1/task.json b/Tasks/DownloadPackageV1/task.json index 2668b303a76d..b5ec321c099e 100644 --- a/Tasks/DownloadPackageV1/task.json +++ b/Tasks/DownloadPackageV1/task.json @@ -10,7 +10,7 @@ "version": { "Major": 1, "Minor": 156, - "Patch": 1 + "Patch": 2 }, "demands": [], "releaseNotes": "Adds support to download Maven, Python, Universal and Npm packages.", @@ -141,9 +141,12 @@ "view": "$(view)", "packageType": "$(packageType)" }, - "endpointUrl": "{{endpoint.url}}/{{ #regex ([a-fA-F0-9\\-]+/)[a-fA-F0-9\\-]+ feed }}_apis/Packaging/Feeds/{{ #regex [a-fA-F0-9\\-]*/([a-fA-F0-9\\-]+) feed }}{{#if view}}@{{{view}}}{{/if}}/Packages?includeUrls=false&isListed=true&includeDeleted=false&protocolType={{{packageType}}}", + "endpointUrl": "{{endpoint.url}}/{{ #regex ([a-fA-F0-9\\-]+/)[a-fA-F0-9\\-]+ feed }}_apis/Packaging/Feeds/{{ #regex [a-fA-F0-9\\-]*/([a-fA-F0-9\\-]+) feed }}{{#if view}}@{{{view}}}{{/if}}/Packages?includeUrls=false&isListed=true&includeDeleted=false&protocolType={{{packageType}}}&$skip={{skip}}&$top=1000", "resultSelector": "jsonpath:$.value[*]", - "resultTemplate": "{ \"Value\" : \"{{{id}}}\", \"DisplayValue\" : \"{{{name}}}\" }" + "resultTemplate": "{ \"Value\" : \"{{{id}}}\", \"DisplayValue\" : \"{{{name}}}\" }", + "callbackContextTemplate": "{\"skip\": \"{{add skip 1000}}\"}", + "callbackRequiredTemplate": "{{isEqualNumber result.count 1000}}", + "initialContextTemplate": "{\"skip\": \"0\"}" }, { "target": "version", diff --git a/Tasks/DownloadPackageV1/task.loc.json b/Tasks/DownloadPackageV1/task.loc.json index f4391ad4785b..654f34389947 100644 --- a/Tasks/DownloadPackageV1/task.loc.json +++ b/Tasks/DownloadPackageV1/task.loc.json @@ -10,7 +10,7 @@ "version": { "Major": 1, "Minor": 156, - "Patch": 1 + "Patch": 2 }, "demands": [], "releaseNotes": "ms-resource:loc.releaseNotes", @@ -141,9 +141,12 @@ "view": "$(view)", "packageType": "$(packageType)" }, - "endpointUrl": "{{endpoint.url}}/{{ #regex ([a-fA-F0-9\\-]+/)[a-fA-F0-9\\-]+ feed }}_apis/Packaging/Feeds/{{ #regex [a-fA-F0-9\\-]*/([a-fA-F0-9\\-]+) feed }}{{#if view}}@{{{view}}}{{/if}}/Packages?includeUrls=false&isListed=true&includeDeleted=false&protocolType={{{packageType}}}", + "endpointUrl": "{{endpoint.url}}/{{ #regex ([a-fA-F0-9\\-]+/)[a-fA-F0-9\\-]+ feed }}_apis/Packaging/Feeds/{{ #regex [a-fA-F0-9\\-]*/([a-fA-F0-9\\-]+) feed }}{{#if view}}@{{{view}}}{{/if}}/Packages?includeUrls=false&isListed=true&includeDeleted=false&protocolType={{{packageType}}}&$skip={{skip}}&$top=1000", "resultSelector": "jsonpath:$.value[*]", - "resultTemplate": "{ \"Value\" : \"{{{id}}}\", \"DisplayValue\" : \"{{{name}}}\" }" + "resultTemplate": "{ \"Value\" : \"{{{id}}}\", \"DisplayValue\" : \"{{{name}}}\" }", + "callbackContextTemplate": "{\"skip\": \"{{add skip 1000}}\"}", + "callbackRequiredTemplate": "{{isEqualNumber result.count 1000}}", + "initialContextTemplate": "{\"skip\": \"0\"}" }, { "target": "version", From 903d1c352064e4307260de9e2078fd24696e2204 Mon Sep 17 00:00:00 2001 From: Aasim Malladi Date: Tue, 23 Jul 2019 16:51:58 -0700 Subject: [PATCH 2/2] Fixed unit tests --- Tasks/DownloadPackageV1/Tests/L0.ts | 24 ++++++++ .../Tests/L0DownloadMultiFilePackage.ts | 2 +- .../Tests/L0DownloadNpmPackage.ts | 2 +- .../Tests/L0DownloadNugetPackage.ts | 2 +- .../L0DownloadNugetPackageNameResolves.ts | 60 +++++++++++++++++++ .../Tests/L0DownloadNugetPackage_noextract.ts | 2 +- .../L0DownloadNugetProjectScopedPackage.ts | 2 +- .../Tests/helpers/webapimock.ts | 15 +++++ 8 files changed, 104 insertions(+), 5 deletions(-) create mode 100644 Tasks/DownloadPackageV1/Tests/L0DownloadNugetPackageNameResolves.ts diff --git a/Tasks/DownloadPackageV1/Tests/L0.ts b/Tasks/DownloadPackageV1/Tests/L0.ts index 6bb5c896e58b..cbd5c6112f5e 100644 --- a/Tasks/DownloadPackageV1/Tests/L0.ts +++ b/Tasks/DownloadPackageV1/Tests/L0.ts @@ -45,6 +45,30 @@ describe("Download single file package suite", function() { done(); }); + it("resolves package id, then downloads and extracts nuget package.", (done: MochaDone) => { + this.timeout(1000); + + let tp: string = path.join(__dirname, "L0DownloadNugetPackageNameResolves.js"); + + let tr: ttm.MockTestRunner = new ttm.MockTestRunner(tp); + + tr.run(); + + assert.equal(tl.ls(null, [tempDir]).length, 1, "should have only 1 file."); + const zipPath = path.join(tempDir, "singlePackageName.nupkg"); + const zipStats = tl.stats(zipPath); + assert(zipStats && zipStats.isFile(), "nupkg file should be downloaded"); + + var extractedFilePath = path.join(destinationDir, "nugetFile"); + const fileStats = tl.stats(extractedFilePath); + assert(fileStats && fileStats.isFile(), "nupkg file should be extracted"); + + assert(tr.stderr.length === 0, "should not have written to stderr"); + assert(tr.succeeded, "task should have succeeded"); + + done(); + }); + it("downloads nuget file from project scoped feed as nupkg and extracts it", (done: MochaDone) => { this.timeout(1000); diff --git a/Tasks/DownloadPackageV1/Tests/L0DownloadMultiFilePackage.ts b/Tasks/DownloadPackageV1/Tests/L0DownloadMultiFilePackage.ts index a2e6c94bc89c..6a4392dc6892 100644 --- a/Tasks/DownloadPackageV1/Tests/L0DownloadMultiFilePackage.ts +++ b/Tasks/DownloadPackageV1/Tests/L0DownloadMultiFilePackage.ts @@ -12,7 +12,7 @@ let tr: tmrm.TaskMockRunner = new tmrm.TaskMockRunner(taskPath); tr.setInput("packageType", "maven"); tr.setInput("feed", "feedId"); tr.setInput("view", "viewId"); -tr.setInput("definition", "packageId"); +tr.setInput("definition", "6f598cbe-a5e2-4f75-aa78-e0fd08301a15"); tr.setInput("version", "versionId"); tr.setInput("downloadPath", outputPath); tr.setInput("files", "*.jar; *.pom"); diff --git a/Tasks/DownloadPackageV1/Tests/L0DownloadNpmPackage.ts b/Tasks/DownloadPackageV1/Tests/L0DownloadNpmPackage.ts index f453da8c111c..cf880d70cfb2 100644 --- a/Tasks/DownloadPackageV1/Tests/L0DownloadNpmPackage.ts +++ b/Tasks/DownloadPackageV1/Tests/L0DownloadNpmPackage.ts @@ -12,7 +12,7 @@ let tr: tmrm.TaskMockRunner = new tmrm.TaskMockRunner(taskPath); tr.setInput("packageType", "npm"); tr.setInput("feed", "feedId"); tr.setInput("view", "viewId"); -tr.setInput("definition", "packageId"); +tr.setInput("definition", "6f598cbe-a5e2-4f75-aa78-e0fd08301a15"); tr.setInput("version", "versionId"); tr.setInput("downloadPath", outputPath); tr.setInput("extract", "true"); diff --git a/Tasks/DownloadPackageV1/Tests/L0DownloadNugetPackage.ts b/Tasks/DownloadPackageV1/Tests/L0DownloadNugetPackage.ts index 261650815183..77194ffc9853 100644 --- a/Tasks/DownloadPackageV1/Tests/L0DownloadNugetPackage.ts +++ b/Tasks/DownloadPackageV1/Tests/L0DownloadNugetPackage.ts @@ -12,7 +12,7 @@ let tr: tmrm.TaskMockRunner = new tmrm.TaskMockRunner(taskPath); tr.setInput("packageType", "nuget"); tr.setInput("feed", "/feedId"); tr.setInput("view", "viewId"); -tr.setInput("definition", "packageId"); +tr.setInput("definition", "6f598cbe-a5e2-4f75-aa78-e0fd08301a15"); tr.setInput("version", "versionId"); tr.setInput("downloadPath", outputPath); tr.setInput("extract", "true"); diff --git a/Tasks/DownloadPackageV1/Tests/L0DownloadNugetPackageNameResolves.ts b/Tasks/DownloadPackageV1/Tests/L0DownloadNugetPackageNameResolves.ts new file mode 100644 index 000000000000..ac384525ed1a --- /dev/null +++ b/Tasks/DownloadPackageV1/Tests/L0DownloadNugetPackageNameResolves.ts @@ -0,0 +1,60 @@ +import tmrm = require("azure-pipelines-task-lib/mock-run"); +import path = require("path"); +import { WebApiMock } from "./helpers/webapimock"; + +let taskPath = path.join(__dirname, "..", "main.js"); +let outputPath: string = path.join(__dirname, "out", "packageOutput"); +let tempPath: string = path.join(__dirname, "temp"); +let zipLocation: string = path.join(tempPath, "singlePackageName.nupkg"); +let tr: tmrm.TaskMockRunner = new tmrm.TaskMockRunner(taskPath); + +// Set inputs +tr.setInput("packageType", "nuget"); +tr.setInput("feed", "/feedId"); +tr.setInput("view", "viewId"); +tr.setInput("definition", "packageName"); +tr.setInput("version", "versionId"); +tr.setInput("downloadPath", outputPath); +tr.setInput("extract", "true"); +tr.setInput("verbosity", "verbose"); + +// Set variables. +process.env["AGENT_TEMPDIRECTORY"] = tempPath; +process.env["SYSTEM_TEAMFOUNDATIONCOLLECTIONURI"] = "https://abc.visualstudio.com/"; +process.env["AGENT_VERSION"] = "2.116.0"; +process.env["HOME"] = "/users/test"; +process.env["ENDPOINT_AUTH_SYSTEMVSSCONNECTION"] = + '{"scheme":"OAuth","parameters":{"AccessToken":"YWFtYWxsYWQ6ZXd0emE1bmN3MzN6c3lyM2NoN2prazUzejczamN6MnluNGtiNzd0ZXc0NnlhZzV2d3ZlcQ=="}}'; + +// provide answers for task mock +tr.setAnswers({ + exist: { + [outputPath]: true, + [tempPath]: true + }, + rmRF: { + [zipLocation]: { + success: true + } + } +}); + +// Register connections mock +tr.registerMock("./connections", { + getConnection: function(): Promise { + return Promise.resolve(new WebApiMock()); + } +}); + +tr.registerMock("./universal", { + downloadUniversalPackage: function( + downloadPath: string, + feedId: string, + packageId: string, + version: string + ): Promise { + return; + } +}); + +tr.run(); diff --git a/Tasks/DownloadPackageV1/Tests/L0DownloadNugetPackage_noextract.ts b/Tasks/DownloadPackageV1/Tests/L0DownloadNugetPackage_noextract.ts index dc5b5e41e936..f6fbffc24f15 100644 --- a/Tasks/DownloadPackageV1/Tests/L0DownloadNugetPackage_noextract.ts +++ b/Tasks/DownloadPackageV1/Tests/L0DownloadNugetPackage_noextract.ts @@ -11,7 +11,7 @@ let tr: tmrm.TaskMockRunner = new tmrm.TaskMockRunner(taskPath); tr.setInput("packageType", "nuget"); tr.setInput("feed", "feedId"); tr.setInput("view", "viewId"); -tr.setInput("definition", "packageId"); +tr.setInput("definition", "6f598cbe-a5e2-4f75-aa78-e0fd08301a15"); tr.setInput("version", "versionId"); tr.setInput("downloadPath", outputPath); tr.setInput("extract", "false"); diff --git a/Tasks/DownloadPackageV1/Tests/L0DownloadNugetProjectScopedPackage.ts b/Tasks/DownloadPackageV1/Tests/L0DownloadNugetProjectScopedPackage.ts index 814d92b2d2d9..96e60db6d3bd 100644 --- a/Tasks/DownloadPackageV1/Tests/L0DownloadNugetProjectScopedPackage.ts +++ b/Tasks/DownloadPackageV1/Tests/L0DownloadNugetProjectScopedPackage.ts @@ -12,7 +12,7 @@ let tr: tmrm.TaskMockRunner = new tmrm.TaskMockRunner(taskPath); tr.setInput("packageType", "nuget"); tr.setInput("feed", "projectId/feedId"); tr.setInput("view", "viewId"); -tr.setInput("definition", "packageId"); +tr.setInput("definition", "6f598cbe-a5e2-4f75-aa78-e0fd08301a15"); tr.setInput("version", "versionId"); tr.setInput("downloadPath", outputPath); tr.setInput("extract", "true"); diff --git a/Tasks/DownloadPackageV1/Tests/helpers/webapimock.ts b/Tasks/DownloadPackageV1/Tests/helpers/webapimock.ts index e2c99f6dcdc6..c2c10de882b9 100644 --- a/Tasks/DownloadPackageV1/Tests/helpers/webapimock.ts +++ b/Tasks/DownloadPackageV1/Tests/helpers/webapimock.ts @@ -35,12 +35,27 @@ class VsoClientMock { routeValues: any, queryParams?: any ): Promise { + if(queryParams && "packageNameQuery" in queryParams) { + return Promise.resolve({ requestUrl: "packageNameResolverUrl" }) + } return Promise.resolve({ requestUrl: this.packageUrlMap[locationId] }); } } class RestMock { private metadataMap = { + packageNameResolverUrl: { + "count":2, + "value": [ + { + "name": "unknown" + }, + { + "name": "packageName", + "id": "6f598cbe-a5e2-4f75-aa78-e0fd08301a15" + } + ] + }, singlePackageMetadataUrl: { name: "singlePackageName" },