DownloadBuildArtifact: FileContainer

Tasks/DownloadBuildArtifact/ArtifactProvider.ts

@@ -0,0 +1,6 @@
+import {BuildArtifact} from 'vso-node-api/interfaces/BuildInterfaces';
+
+export interface ArtifactProvider {
+    supportsArtifactType(artifactType: string): boolean;
+    downloadArtifact(artifact: BuildArtifact, targetPath: string): Promise<void>;
+}

Tasks/DownloadBuildArtifact/Downloader.ts

@@ -0,0 +1,84 @@
+import * as path from 'path';
+import * as tl from 'vsts-task-lib/task';
+import * as fs from 'fs';
+
+/**
+ * Represents an item to be downloaded
+ */
+export interface DownloadItem<T> {
+    /**
+     * The path to the item, relative to the target path
+     */
+    relativePath: string;
+
+    /**
+     * Artifact-specific data
+     */
+    data: T;
+}
+
+/**
+ * Downloads items
+ * @param items the items to download
+ * @param targetPath the folder that will hold the downloaded items
+ * @param maxConcurrency the maximum number of items to download simultaneously
+ * @param downloader a function that, given a DownloadItem, will return a content stream
+ */
+export async function download<T>(items: DownloadItem<T>[], targetPath: string, maxConcurrency: number, downloader: (item: DownloadItem<T>) => Promise<fs.ReadStream>): Promise<void> {
+    // keep track of folders we've touched so we don't call mkdirP for every single file
+    let createdFolders: { [key: string]: boolean } = {};
+    let downloaders: Promise<{}>[] = [];
+    let fileCount: number = items.length;
+
+    maxConcurrency = Math.min(maxConcurrency, items.length);
+    for (let i = 0; i < maxConcurrency; ++i) {
+        downloaders.push(new Promise(async (resolve, reject) => {
+            try {
+                while (items.length > 0) {
+                    let item = items.pop();
+                    let fileIndex = fileCount - items.length;
+
+                    // the full path of the downloaded file
+                    let outputFilename = path.join(targetPath, item.relativePath);
+
+                    // create the folder if necessary
+                    let folder = path.dirname(outputFilename);
+                    if (!createdFolders.hasOwnProperty(folder)) {
+                        if (!tl.exist(folder)) {
+                            tl.mkdirP(folder);
+                        }
+                        createdFolders[folder] = true;
+                    }
+
+                    console.log(tl.loc("DownloadingFile", item.relativePath, outputFilename, fileIndex, fileCount));
+                    await new Promise(async (downloadResolve, downloadReject) => {
+                        try {
+                            // get the content stream from the provider
+                            let contentStream = await downloader(item);
+
+                            // create the target stream
+                            let outputStream = fs.createWriteStream(outputFilename);
+
+                            // pipe the content to the target
+                            contentStream.pipe(outputStream);
+                            contentStream.on('end', () => {
+                                tl.debug(`Downloaded '${item.relativePath}' to '${outputFilename}'`);
+                                downloadResolve();
+                            });
+                        }
+                        catch (err) {
+                            console.log(tl.loc("DownloadError", item.relativePath));
+                            downloadReject(err);
+                        }
+                    });
+                }
+                resolve();
+            }
+            catch (err) {
+                reject(err);
+            }
+        }));
+    }
+
+    await Promise.all(downloaders);
+}

Tasks/DownloadBuildArtifact/FileContainer.ts

@@ -0,0 +1,80 @@
+import * as tl from 'vsts-task-lib/task';
+import * as fs from 'fs';
+import * as path from 'path';
+
+import {BuildArtifact} from 'vso-node-api/interfaces/BuildInterfaces';
+import {FileContainerItem, ContainerItemType} from 'vso-node-api/interfaces/FileContainerInterfaces';
+import {IFileContainerApi} from 'vso-node-api/FileContainerApi';
+import {WebApi, getHandlerFromToken} from 'vso-node-api/WebApi';
+
+import {DownloadItem, download} from './Downloader';
+import {ArtifactProvider} from './ArtifactProvider';
+
+export class FileContainerProvider implements ArtifactProvider {
+    public supportsArtifactType(artifactType: string): boolean {
+        return !!artifactType && artifactType.toLowerCase() === "container";
+    }
+
+    public async downloadArtifact(artifact: BuildArtifact, targetPath: string): Promise<void> {
+        if (!artifact || !artifact.resource || !artifact.resource.data) {
+            throw new Error(tl.loc("FileContainerInvalidArtifact"));
+        }
+
+        let containerParts: string[] = artifact.resource.data.split('/', 3);
+        if (containerParts.length !== 3) {
+            throw new Error(tl.loc("FileContainerInvalidArtifactData"));
+        }
+
+        let containerId: number = parseInt(containerParts[1]);
+        let containerPath: string = containerParts[2];
+
+        let accessToken = tl.getEndpointAuthorizationParameter('SYSTEMVSSCONNECTION', 'AccessToken', false);
+		let credentialHandler = getHandlerFromToken(accessToken);
+        let collectionUrl = tl.getEndpointUrl('SYSTEMVSSCONNECTION', false);
+        let vssConnection = new WebApi(collectionUrl, credentialHandler);
+
+        let fileContainerApi = vssConnection.getFileContainerApi();
+
+        // get all items
+        let items: FileContainerItem[] = await fileContainerApi.getItems(containerId, null, containerPath, false, null, null, false, false);
+
+        // ignore folders
+        items = items.filter(item => item.itemType === ContainerItemType.File);
+        tl.debug(`Found ${items.length} File items in #/${containerId}/${containerPath}`);
+
+        let downloadItems: DownloadItem<FileContainerItem>[] = items.map((item) => {
+            return { 
+                relativePath: item.path,
+                data: item
+            };
+        })
+
+        // download the items
+        await download(downloadItems, targetPath, 8, (item: DownloadItem<FileContainerItem>) => new Promise(async (resolve, reject) => {
+            try {
+                let downloadFilename = item.data.path.substring(item.data.path.lastIndexOf("/") + 1);
+                let itemResponse = await fileContainerApi.getItem(containerId, null, item.data.path, downloadFilename);
+                if (itemResponse.statusCode === 200) {
+                    resolve(itemResponse.result);
+                }
+                else {
+                    // TODO: decide whether to retry or bail
+                    reject(itemResponse);
+                }
+            }
+            catch (err) {
+                reject(err);
+            }
+        }));
+    }
+}
+
+function getAuthToken() {
+    let auth = tl.getEndpointAuthorization('SYSTEMVSSCONNECTION', false);
+    if (auth.scheme.toLowerCase() === 'oauth') {
+        return auth.parameters['AccessToken'];
+    }
+    else {
+        throw new Error(tl.loc("CredentialsNotFound"))
+    }
+}

Tasks/DownloadBuildArtifact/Strings/resources.resjson/en-US/resources.resjson

@@ -0,0 +1,19 @@
+{
+  "loc.friendlyName": "Download Build Artifact",
+  "loc.helpMarkDown": "",
+  "loc.description": "Download a build artifact.",
+  "loc.instanceNameFormat": "Download build artifact $(artifactName)",
+  "loc.input.label.buildId": "Build",
+  "loc.input.help.buildId": "The build from which to download the artifact",
+  "loc.input.label.artifactName": "Artifact name",
+  "loc.input.help.artifactName": "The name of the artifact to download",
+  "loc.input.label.downloadPath": "Destination directory",
+  "loc.input.help.downloadPath": "Path on the agent machine where the artifact will be downloaded",
+  "loc.messages.FileContainerCredentialsNotFound": "Could not determine credentials to connect to file container service.",
+  "loc.messages.FileContainerInvalidArtifact": "Invalid file container artifact",
+  "loc.messages.FileContainerInvalidArtifactData": "Invalid file container artifact. Resource data must be in the format #/{container id}/path",
+  "loc.messages.ArtifactProviderNotFound": "Could not determine a provider to download artifact of type {0}",
+  "loc.messages.DownloadError": "Error downloading file {0}",
+  "loc.messages.DownloadingFile": "Downloading '{0}' to '{1}' (file {2}/{3})",
+  "loc.messages.InvalidBuildId": "Invalid build id specified ({0})"
+}

Tasks/DownloadBuildArtifact/main.ts

@@ -0,0 +1,47 @@
+import {BuildArtifact, ArtifactResource} from 'vso-node-api/interfaces/BuildInterfaces';
+import {WebApi, getHandlerFromToken} from 'vso-node-api/WebApi';
+import * as tl from 'vsts-task-lib/task';
+
+import {ArtifactProvider} from './ArtifactProvider';
+import {FileContainerProvider} from './FileContainer';
+
+async function main(): Promise<void> {
+	try {
+		let projectId = tl.getVariable('System.TeamProjectId');
+		let artifactName = tl.getInput("artifactName");
+		let downloadPath = tl.getPathInput("downloadPath");
+		let buildId = parseInt(tl.getInput("buildId"));
+
+		if (isNaN(buildId)) {
+			throw new Error(tl.loc("InvalidBuildId", tl.getInput("buildId")));
+		}
+
+		let accessToken = tl.getEndpointAuthorizationParameter('SYSTEMVSSCONNECTION', 'AccessToken', false);
+		let credentialHandler = getHandlerFromToken(accessToken);
+		let collectionUrl = tl.getEndpointUrl('SYSTEMVSSCONNECTION', false);
+		let vssConnection = new WebApi(collectionUrl, credentialHandler);
+
+		// get the artifact metadata
+		let buildApi = vssConnection.getBuildApi();
+		let artifact = await buildApi.getArtifact(buildId, artifactName, projectId);
+
+		let providers: ArtifactProvider[] = [
+			new FileContainerProvider()
+		];
+
+		let provider = providers.filter((provider) => provider.supportsArtifactType(artifact.resource.type))[0];
+		if (provider) {
+			await provider.downloadArtifact(artifact, downloadPath);
+		}
+		else {
+			throw new Error(tl.loc("ArtifactProviderNotFound", artifact.resource.type));
+		}
+
+		tl.setResult(tl.TaskResult.Succeeded, "");
+	}
+	catch (err) {
+		tl.setResult(tl.TaskResult.Failed, err);
+	}
+}
+
+main();

Tasks/DownloadBuildArtifact/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "downloadbuildartifact",
+  "version": "0.1.0",
+  "description": "Download Build Artifact Task",
+  "main": "download.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Microsoft/vsts-tasks.git"
+  },
+  "author": "Microsoft Corporation",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/Microsoft/vsts-tasks/issues"
+  },
+  "homepage": "https://github.com/Microsoft/vsts-tasks#readme",
+  "dependencies": {
+    "@types/node": "^6.0.78",
+    "vso-node-api": "^6.2.5-preview",
+    "vsts-task-lib": "^2.0.3-preview"
+  }
+}

Tasks/DownloadBuildArtifact/task.json

@@ -0,0 +1,63 @@
+{
+  "id": "a433f589-fce1-4460-9ee6-44a624aeb1fb",
+  "name": "DownloadBuildArtifact",
+  "friendlyName": "Download Build Artifact",
+  "description": "Download a build artifact.",
+  "helpMarkDown": "",
+  "category": "Utility",
+  "author": "Microsoft Corporation",
+  "preview": true,
+  "version": {
+    "Major": 0,
+    "Minor": 1,
+    "Patch": 48
+  },
+  "demands": [],
+  "inputs": [
+    {
+      "name": "buildId",
+      "type": "pickList",
+      "label": "Build",
+      "required": false,
+      "helpMarkDown": "The build from which to download the artifact",
+      "defaultValue": "$(Build.BuildId)",
+      "options": {
+        "$(Build.BuildId)": "The current build"
+      }
+    },
+    {
+      "name": "artifactName",
+      "type": "string",
+      "label": "Artifact name",
+      "defaultValue": "drop",
+      "required": true,
+      "helpMarkDown": "The name of the artifact to download"
+    },
+    {
+      "name": "downloadPath",
+      "type": "string",
+      "label": "Destination directory",
+      "defaultValue": "$(System.ArtifactsDirectory)",
+      "required": true,
+      "helpMarkDown": "Path on the agent machine where the artifact will be downloaded"
+    }
+  ],
+  "dataSourceBindings": [
+  ],
+  "instanceNameFormat": "Download build artifact $(artifactName)",
+  "execution": {
+    "Node": {
+      "target": "main.js",
+      "argumentFormat": ""
+    }
+  },
+  "messages": {
+    "FileContainerCredentialsNotFound": "Could not determine credentials to connect to file container service.",
+    "FileContainerInvalidArtifact": "Invalid file container artifact",
+    "FileContainerInvalidArtifactData": "Invalid file container artifact. Resource data must be in the format #/{container id}/path",
+    "ArtifactProviderNotFound": "Could not determine a provider to download artifact of type {0}",
+    "DownloadError": "Error downloading file {0}",
+    "DownloadingFile": "Downloading '{0}' to '{1}' (file {2}/{3})",
+    "InvalidBuildId": "Invalid build id specified ({0})"
+  }
+}

Tasks/DownloadBuildArtifact/task.loc.json

@@ -0,0 +1,62 @@
+{
+  "id": "a433f589-fce1-4460-9ee6-44a624aeb1fb",
+  "name": "DownloadBuildArtifact",
+  "friendlyName": "ms-resource:loc.friendlyName",
+  "description": "ms-resource:loc.description",
+  "helpMarkDown": "ms-resource:loc.helpMarkDown",
+  "category": "Utility",
+  "author": "Microsoft Corporation",
+  "preview": true,
+  "version": {
+    "Major": 0,
+    "Minor": 1,
+    "Patch": 48
+  },
+  "demands": [],
+  "inputs": [
+    {
+      "name": "buildId",
+      "type": "pickList",
+      "label": "ms-resource:loc.input.label.buildId",
+      "required": false,
+      "helpMarkDown": "ms-resource:loc.input.help.buildId",
+      "defaultValue": "$(Build.BuildId)",
+      "options": {
+        "$(Build.BuildId)": "The current build"
+      }
+    },
+    {
+      "name": "artifactName",
+      "type": "string",
+      "label": "ms-resource:loc.input.label.artifactName",
+      "defaultValue": "drop",
+      "required": true,
+      "helpMarkDown": "ms-resource:loc.input.help.artifactName"
+    },
+    {
+      "name": "downloadPath",
+      "type": "string",
+      "label": "ms-resource:loc.input.label.downloadPath",
+      "defaultValue": "$(System.ArtifactsDirectory)",
+      "required": true,
+      "helpMarkDown": "ms-resource:loc.input.help.downloadPath"
+    }
+  ],
+  "dataSourceBindings": [],
+  "instanceNameFormat": "ms-resource:loc.instanceNameFormat",
+  "execution": {
+    "Node": {
+      "target": "main.js",
+      "argumentFormat": ""
+    }
+  },
+  "messages": {
+    "FileContainerCredentialsNotFound": "ms-resource:loc.messages.FileContainerCredentialsNotFound",
+    "FileContainerInvalidArtifact": "ms-resource:loc.messages.FileContainerInvalidArtifact",
+    "FileContainerInvalidArtifactData": "ms-resource:loc.messages.FileContainerInvalidArtifactData",
+    "ArtifactProviderNotFound": "ms-resource:loc.messages.ArtifactProviderNotFound",
+    "DownloadError": "ms-resource:loc.messages.DownloadError",
+    "DownloadingFile": "ms-resource:loc.messages.DownloadingFile",
+    "InvalidBuildId": "ms-resource:loc.messages.InvalidBuildId"
+  }
+}