From 3a20113b52e3af3804bf5905bcce19d8a01d6f73 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Mon, 13 Jan 2020 11:07:47 -0500 Subject: [PATCH 01/46] Initial commit for @actions/artifact --- packages/artifact/README.md | 0 packages/artifact/package-lock.json | 54 ++++ packages/artifact/package.json | 43 +++ packages/artifact/src/artifact.ts | 111 +++++++ packages/artifact/src/contracts.ts | 28 ++ packages/artifact/src/download-info.ts | 11 + packages/artifact/src/search.ts | 95 ++++++ .../src/upload-artifact-http-client.ts | 300 ++++++++++++++++++ packages/artifact/src/upload-info.ts | 16 + packages/artifact/src/utils.ts | 94 ++++++ packages/artifact/tsconfig.json | 11 + 11 files changed, 763 insertions(+) create mode 100644 packages/artifact/README.md create mode 100644 packages/artifact/package-lock.json create mode 100644 packages/artifact/package.json create mode 100644 packages/artifact/src/artifact.ts create mode 100644 packages/artifact/src/contracts.ts create mode 100644 packages/artifact/src/download-info.ts create mode 100644 packages/artifact/src/search.ts create mode 100644 packages/artifact/src/upload-artifact-http-client.ts create mode 100644 packages/artifact/src/upload-info.ts create mode 100644 packages/artifact/src/utils.ts create mode 100644 packages/artifact/tsconfig.json diff --git a/packages/artifact/README.md b/packages/artifact/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/artifact/package-lock.json b/packages/artifact/package-lock.json new file mode 100644 index 0000000000..3c5cd5d048 --- /dev/null +++ b/packages/artifact/package-lock.json @@ -0,0 +1,54 @@ +{ + "name": "@actions/artifact", + "version": "0.1.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@actions/core": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.2.1.tgz", + "integrity": "sha512-xD+CQd9p4lU7ZfRqmUcbJpqR+Ss51rJRVeXMyOLrZQImN9/8Sy/BEUBnHO/UKD3z03R686PVTLfEPmkropGuLw==" + }, + "@actions/glob": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@actions/glob/-/glob-0.1.0.tgz", + "integrity": "sha512-lx8SzyQ2FE9+UUvjqY1f28QbTJv+w8qP7kHHbfQRhphrlcx0Mdmm1tZdGJzfxv1jxREa/sLW4Oy8CbGQKCJySA==", + "requires": { + "@actions/core": "^1.2.0", + "minimatch": "^3.0.4" + } + }, + "@actions/http-client": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-1.0.1.tgz", + "integrity": "sha512-vy5DhqTJ1gtEkpRrD/6BHhUlkeyccrOX0BT9KmtO5TWxe5KSSwVHFE+J15Z0dG+tJwZJ/nHC4slUIyqpkahoMg==" + }, + "balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "requires": { + "brace-expansion": "^1.1.7" + } + } + } +} diff --git a/packages/artifact/package.json b/packages/artifact/package.json new file mode 100644 index 0000000000..b2ed07a5dd --- /dev/null +++ b/packages/artifact/package.json @@ -0,0 +1,43 @@ +{ + "name": "@actions/artifact", + "version": "0.1.0", + "preview": true, + "description": "Actions artifact lib", + "keywords": [ + "github", + "actions", + "artifact" + ], + "homepage": "https://github.com/actions/toolkit/tree/master/packages/artifact", + "license": "MIT", + "main": "lib/artifact.js", + "types": "lib/artifact.d.ts", + "directories": { + "lib": "lib", + "test": "__tests__" + }, + "files": [ + "lib" + ], + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/actions/toolkit.git", + "directory": "packages/artifact" + }, + "scripts": { + "audit-moderate": "npm install && npm audit --audit-level=moderate", + "test": "echo \"Error: no test specified\" && exit 1", + "tsc": "tsc" + }, + "bugs": { + "url": "https://github.com/actions/toolkit/issues" + }, + "dependencies": { + "@actions/core": "^1.2.1", + "@actions/glob": "^0.1.0", + "@actions/http-client": "^1.0.1" + } +} diff --git a/packages/artifact/src/artifact.ts b/packages/artifact/src/artifact.ts new file mode 100644 index 0000000000..e0609016dc --- /dev/null +++ b/packages/artifact/src/artifact.ts @@ -0,0 +1,111 @@ +import {UploadInfo} from './upload-info' +import { + createArtifactInFileContainer, + uploadArtifactToFileContainer, + patchArtifactSize +} from './upload-artifact-http-client' +import {checkArtifactName} from './utils' +import * as core from '@actions/core' +import {CreateArtifactResponse} from './contracts' +import {SearchResult, findFilesToUpload} from './search' + +/** + * Uploads an artifact + * + * @param name the name of the artifact, required + * @param path the directory, file, or glob pattern to denote what will be uploaded, required + * @returns single UploadInfo object + */ +export async function uploadArtifact( + name: string, + path: string +): Promise { + // Check that the required inputs are valid + if (!name) { + throw new Error('Artifact name must be provided') + } + + checkArtifactName(name) + + if (!path) { + throw new Error('Upload path must be provided') + } + + // Search for the items that will be uploaded + let filesToUpload: SearchResult[] = await findFilesToUpload(name, path) + + if (filesToUpload === undefined) { + core.setFailed( + `Unable to succesfully search fo files to upload with the provided path: ${path}` + ) + } else if (filesToUpload.length == 0) { + core.warning( + `No files were found for the provided path: ${path}. No artifacts will be uploaded.` + ) + } else { + /** + * Step 1 of 3 + * Create an entry for the artifact in the file container + */ + const response: CreateArtifactResponse = await createArtifactInFileContainer( + name + ) + if (!response.fileContainerResourceUrl) { + console.log(response) + throw new Error('Unable to get fileContainerResourceUrl') + } + core.debug(`We will be uploading to: ${response.fileContainerResourceUrl}`) + + /** + * Step 2 of 3 + * Upload each of the files that were found concurrently + */ + let artfactSize: Promise = Promise.resolve( + uploadArtifactToFileContainer( + response.fileContainerResourceUrl!, + filesToUpload + ) + ) + artfactSize.then(async size => { + console.log(`Size of what we just uploaded is ${size}`) + /** + * Step 3 of 3 + * Update the size of the artifact to indicate we are done uploading + */ + await patchArtifactSize(size, name) + return { + artifactName: name, + artifactItems: filesToUpload.map(item => item.absoluteFilePath), + size: size + } as UploadInfo + }) + } + + return { + artifactName: name, + artifactItems: filesToUpload.map(item => item.absoluteFilePath), + size: -1 + } as UploadInfo +} + +/* +Downloads a single artifact associated with a run + +export async function downloadArtifact( + name: string, + path?: string, + createArtifactFolder?:boolean + ): Promise { + + TODO +} + +Downloads all artifacts associated with a run. Because there are multiple artifacts being downloaded, a folder will be created for each one in the specified or default directory + +export async function downloadAllArtifacts( + path?: string + ): Promise{ + + TODO +} +*/ diff --git a/packages/artifact/src/contracts.ts b/packages/artifact/src/contracts.ts new file mode 100644 index 0000000000..b0bbaa9434 --- /dev/null +++ b/packages/artifact/src/contracts.ts @@ -0,0 +1,28 @@ +export interface CreateArtifactResponse { + containerId?: string + size?: number + signedContent?: string + fileContainerResourceUrl?: string + type?: string + name?: string + url?: string +} + +export interface CreateArtifactParameters { + Type: string + Name: string +} + +export interface PatchArtifactSize { + Size: number +} + +export interface PatchArtifactSizeSuccessResponse { + containerId: number + size: number + signedContent: string + type: string + name: string + url: string + uploadUrl: string +} diff --git a/packages/artifact/src/download-info.ts b/packages/artifact/src/download-info.ts new file mode 100644 index 0000000000..35b88c24f2 --- /dev/null +++ b/packages/artifact/src/download-info.ts @@ -0,0 +1,11 @@ +export interface DownloadInfo { + /** + * The name of the artifact that was downloaded + */ + artifactName: string + + /** + * The full Path to where the artifact was downloaded + */ + downloadPath: string +} diff --git a/packages/artifact/src/search.ts b/packages/artifact/src/search.ts new file mode 100644 index 0000000000..3c15ae1d09 --- /dev/null +++ b/packages/artifact/src/search.ts @@ -0,0 +1,95 @@ +import * as glob from '@actions/glob' +import {join, basename} from 'path' +import {debug} from '@actions/core' +import {lstatSync} from 'fs' + +export interface SearchResult { + absoluteFilePath: string + uploadFilePath: string +} + +/** + * Searches the provided path and returns the files that will be uploaded as part of the artifact + * @param {string} searchPath Wildcard pattern, directory or individual file that is provided by the user to specify what should be uploaded + * @param {string} artifactName The name of the artifact, used as the root directory when uploading artifacts to glob storage + * @return A list of files that should be uploaded along with the paths they will be uploaded with + */ +export async function findFilesToUpload( + artifactName: string, + searchPath: string +): Promise { + let searchResults: SearchResult[] = [] + let itemsToUpload: string[] = [] + let options: glob.GlobOptions = { + followSymbolicLinks: true, + implicitDescendants: true, + omitBrokenSymbolicLinks: true + } + let globber = await glob.create(searchPath, options) + + let rawSearchResults: string[] = await globber.glob() + /** + * Directories will be rejected if attempted to be uploaded. This includes just empty + * directories so filter any directories out from the raw search results + */ + + rawSearchResults.forEach(function(item) { + if (!lstatSync(item).isDirectory()) { + itemsToUpload.push(item) + } else { + debug(`Removing ${item} from rawSearchResults because it is a directory`) + } + }) + + if (itemsToUpload.length == 0) { + return searchResults + } + console.log( + `Found the following ${itemsToUpload.length} items that will be uploaded as part of the artifact:` + ) + console.log(itemsToUpload) + + /** + * Only a single search pattern is being included so only 1 searchResult is expected. In the future if multiple search patterns are + * simultaneously supported this will change + */ + let searchPaths: string[] = await globber.getSearchPaths() + if (searchResults.length > 1) { + console.log(searchResults) + throw new Error('Only 1 search path should be returned') + } + + /** + * Creates the path that the artifact will be uploaded with. The artifact name will always be the root directory so that + * it can be distinguished from other artifacts that are uploaded to the same file Container/glob storage during a run + */ + if (itemsToUpload.length == 1) { + // A single artifact will be uploaded, the upload path will always be in the form ${artifactName}\${singleArtifactName} + searchResults.push({ + absoluteFilePath: itemsToUpload[0], + uploadFilePath: join(artifactName, basename(searchPaths[0])) + }) + } else { + /** + * multiple files will be uploaded as part of the artifact + * The search path denotes the base path that was used to find the file. It will be removed from the absolute path and + * the artifact name will be prepended to create the path used during upload + */ + itemsToUpload.forEach(function(file) { + let uploadPath: string = file.replace(searchPaths[0], '') + searchResults.push({ + absoluteFilePath: file, + uploadFilePath: artifactName.concat(uploadPath) + }) + }) + } + + debug('SearchResult includes the following information:') + searchResults.forEach(function(item) { + debug( + `Absolute File Path: ${item.absoluteFilePath}\nUpload file path: ${item.uploadFilePath}` + ) + }) + + return searchResults +} diff --git a/packages/artifact/src/upload-artifact-http-client.ts b/packages/artifact/src/upload-artifact-http-client.ts new file mode 100644 index 0000000000..8b705e4a5f --- /dev/null +++ b/packages/artifact/src/upload-artifact-http-client.ts @@ -0,0 +1,300 @@ +import {debug} from '@actions/core' +import * as fs from 'fs' +import {HttpClientResponse, HttpClient} from '@actions/http-client/index' +import {BearerCredentialHandler} from '@actions/http-client/auth' +import { + CreateArtifactResponse, + CreateArtifactParameters, + PatchArtifactSize, + PatchArtifactSizeSuccessResponse +} from './contracts' +import { + parseEnvNumber, + getArtifactUrl, + isSuccessStatusCode, + isRetryableStatusCode, + getRequestOptions, + getContentRange +} from './utils' +import {SearchResult} from './search' +import {IHttpClientResponse} from '@actions/http-client/interfaces' +import {URL} from 'url' + +const defaultChunkUploadConcurrency = 3 +const defaultFileUploadConcurrency = 2 + +/** + * Step 1 of 3 when uploading an artifact. Creates a file container for the new artifact in the remote blob storage/file service + * @param {string} artifactName Name of the artifact being created + * @returns The response from the Artifact Service if the file container was succesfully created + */ +export async function createArtifactInFileContainer( + artifactName: string +): Promise { + const token = process.env['ACTIONS_RUNTIME_TOKEN'] || '' + const bearerCredentialHandler = new BearerCredentialHandler(token) + const requestOptions = getRequestOptions() + requestOptions['Content-Type'] = 'application/json' + + let client: HttpClient = new HttpClient('actions/artifact', [ + bearerCredentialHandler + ]) + let parameters: CreateArtifactParameters = { + Type: 'actions_storage', + Name: artifactName + } + let data: string = JSON.stringify(parameters, null, 2) + + const artifactUrl: string = getArtifactUrl() + const rawResponse: HttpClientResponse = await client.post( + artifactUrl, + data, + requestOptions + ) + let body: string = await rawResponse.readBody() + let response: CreateArtifactResponse = JSON.parse(body) + console.log(response) + + if (rawResponse.message.statusCode == 201 && response) { + return response + } else { + throw new Error( + 'Non 201 status code when creating file container for new artifact' + ) + } +} + +/** + * Step 2 of 3 when uploading an artifact. Concurrently upload all of the files in chunks + * @param {string} uploadUrl Base Url for the artifact that was created + * @param {SearchResult[]} filesToUpload A list of information about the files being uploaded + * @returns The size of all the files uploaded in bytes + */ +export async function uploadArtifactToFileContainer( + uploadUrl: string, + filesToUpload: SearchResult[] +): Promise { + const token = process.env['ACTIONS_RUNTIME_TOKEN'] || '' + const bearerCredentialHandler = new BearerCredentialHandler(token) + let client: HttpClient = new HttpClient('actions/artifact', [ + bearerCredentialHandler + ]) + + const FILE_CONCURRENCY = + parseEnvNumber('ARTIFACT_FILE_UPLOAD_CONCURRENCY') || + defaultFileUploadConcurrency + const CHUNK_CONCURRENCY = + parseEnvNumber('ARTIFACT_CHUNK_UPLOAD_CONCURRENCY') || + defaultChunkUploadConcurrency + const MAX_CHUNK_SIZE = + parseEnvNumber('ARTIFACT_UPLOAD_CHUNK_SIZE') || 4 * 1024 * 1024 // 4 MB Chunks + debug( + `File Concurrency: ${FILE_CONCURRENCY}, Chunk Concurrency: ${CHUNK_CONCURRENCY} and Chunk Size: ${MAX_CHUNK_SIZE}` + ) + + let parameters: UploadFileParameters[] = [] + + // Prepare the necessary parameters to upload all the files + filesToUpload.forEach(function(item) { + let resourceUrl = new URL(uploadUrl) + resourceUrl.searchParams.append( + 'scope', + '00000000-0000-0000-0000-000000000000' + ) + resourceUrl.searchParams.append('itemPath', item.uploadFilePath) + parameters.push({ + file: item.absoluteFilePath, + resourceUrl: resourceUrl.toString(), + restClient: client, + concurrency: CHUNK_CONCURRENCY, + maxChunkSize: MAX_CHUNK_SIZE + }) + }) + + const parallelUploads = [...new Array(FILE_CONCURRENCY).keys()] + let uploadedFiles = 0 + let fileSizes: number[] = [] + + // Only allow a certain amount of files to be uploaded at once, this is done to reduce errors if + // trying to upload everything at once + await Promise.all( + parallelUploads.map(async () => { + while (uploadedFiles < filesToUpload.length) { + let currentFileParameters = parameters[uploadedFiles] + uploadedFiles += 1 + fileSizes.push(await uploadFileAsync(currentFileParameters)) + } + }) + ) + + let sum: number = 0 + for (var i = 0; i < fileSizes.length; i++) { + sum += fileSizes[i] + } + console.log(`Total size of all the files uploaded ${sum}`) + return sum +} + +/** + * Asyncronously uploads a file. If the file is bigger than the max chunk size it will be uploaded via multiple calls + * @param {UploadFileParameters} parameters Information about the files that need to be uploaded + * @returns The size of the file that was uploaded in bytes + */ +async function uploadFileAsync( + parameters: UploadFileParameters +): Promise { + const fileSize: number = fs.statSync(parameters.file).size + const parallelUploads = [...new Array(parameters.concurrency).keys()] + let offset = 0 + + await Promise.all( + parallelUploads.map(async () => { + while (offset < fileSize) { + const chunkSize = Math.min(fileSize - offset, parameters.maxChunkSize) + const start = offset + const end = offset + chunkSize - 1 + offset += parameters.maxChunkSize + const chunk: NodeJS.ReadableStream = fs.createReadStream( + parameters.file, + { + start, + end, + autoClose: false + } + ) + + await uploadChunk( + parameters.restClient, + parameters.resourceUrl, + chunk, + start, + end, + fileSize + ) + } + }) + ) + return fs.statSync(parameters.file).size +} + +/** + * Uploads a chunk of an individual file to the specified resourceUrl. If the upload fails and the status code + * indicates a retryable status, we try to upload the chunk as well + * @param {HttpClient} restClient RestClient that will be making the appropriate HTTP call + * @param {string} resourceUrl Url of the resource that the chunk will be uploaded to + * @param {NodeJS.ReadableStream} data Stream of the file that will be uploaded + * @param {number} start Starting byte index of file that the chunk belongs to + * @param {number} end Ending byte index of file that the chunk belongs to + * @param {number} totalSize Total size of the file in bytes that is being uploaded + */ +async function uploadChunk( + restClient: HttpClient, + resourceUrl: string, + data: NodeJS.ReadableStream, + start: number, + end: number, + totalSize: number +): Promise { + console.log( + `Uploading chunk of size ${end - + start + + 1} bytes at offset ${start} with content range: ${getContentRange( + start, + end, + totalSize + )}` + ) + + const requestOptions = getRequestOptions() + requestOptions['Content-Type'] = 'application/octet-stream' + requestOptions['Content-Length'] = totalSize + requestOptions['Content-Range'] = getContentRange(start, end, totalSize) + + const uploadChunkRequest = async (): Promise => { + return await restClient.sendStream('PUT', resourceUrl, data, requestOptions) + } + + const response = await uploadChunkRequest() + + if (!response.message.statusCode) { + console.log(response) + throw new Error('No Status Code returned with response') + } + + if (isSuccessStatusCode(response.message.statusCode)) { + debug( + `Chunk for ${start}:${end} was succesfully uploaded to ${resourceUrl}` + ) + return + } + if (isRetryableStatusCode(response.message.statusCode)) { + console.log( + `Received ${response.message.statusCode}, will retry chunk at offset ${start} after 10 seconds.` + ) + await new Promise(resolve => setTimeout(resolve, 10000)) + console.log(`Retrying chunk at offset ${start}`) + + const retryResponse = await uploadChunkRequest() + if (!retryResponse.message.statusCode) { + console.log(retryResponse) + throw new Error('No Status Code returne with response') + } + if (isSuccessStatusCode(retryResponse.message.statusCode)) { + return + } + } +} + +/** + * Step 3 of 3 when uploading an artifact + * Updates the size of the artifact from -1 which was initially set during step 1. Updating the size indicates that we are + * done uploading all the contents of the artifact. A server side check will be run to check that the artifact size is correct + * for billing purposes + */ +export async function patchArtifactSize( + size: number, + artifactName: string +): Promise { + const requestOptions = getRequestOptions() + requestOptions['Content-Type'] = 'application/json' + + let resourceUrl = new URL(getArtifactUrl()) + resourceUrl.searchParams.append('artifactName', artifactName) + + let parameters: PatchArtifactSize = {Size: size} + let data: string = JSON.stringify(parameters, null, 2) + + const token = process.env['ACTIONS_RUNTIME_TOKEN'] || '' + const bearerCredentialHandler = new BearerCredentialHandler(token) + let client: HttpClient = new HttpClient('actions/artifact', [ + bearerCredentialHandler + ]) + + console.log(`URL is ${resourceUrl.toString()}`) + + let rawResponse: HttpClientResponse = await client.patch( + resourceUrl.toString(), + data, + requestOptions + ) + let body: string = await rawResponse.readBody() + + if (rawResponse.message.statusCode == 200) { + let successResponse: PatchArtifactSizeSuccessResponse = JSON.parse(body) + console.log('Artifact size was succesfully updated!') + console.log(successResponse) + } else if (rawResponse.message.statusCode == 404) { + throw new Error(`An Artifact with the name ${artifactName} was not found`) + } else { + console.log(body) + throw new Error('Unable to update the artifact size') + } +} + +interface UploadFileParameters { + file: string + resourceUrl: string + restClient: HttpClient + concurrency: number + maxChunkSize: number +} diff --git a/packages/artifact/src/upload-info.ts b/packages/artifact/src/upload-info.ts new file mode 100644 index 0000000000..7aa1d7d49f --- /dev/null +++ b/packages/artifact/src/upload-info.ts @@ -0,0 +1,16 @@ +export interface UploadInfo { + /** + * The name of the artifact that was uploaded + */ + artifactName: string + + /** + * A list of items that were uploaded as part of the artifact + */ + artifactItems: string[] + + /** + * Total size of the artifact in bytes that was uploaded + */ + size: number +} diff --git a/packages/artifact/src/utils.ts b/packages/artifact/src/utils.ts new file mode 100644 index 0000000000..b52fd16975 --- /dev/null +++ b/packages/artifact/src/utils.ts @@ -0,0 +1,94 @@ +import {HttpCodes, HttpClient} from '@actions/http-client' +import {debug} from '@actions/core' +import {IHeaders} from '@actions/http-client/interfaces' + +const apiVersion: string = '6.0-preview' + +/** + * Parses a env variable that is a number + */ +export function parseEnvNumber(key: string): number | undefined { + const value = Number(process.env[key]) + if (Number.isNaN(value) || value < 0) { + return undefined + } + return value +} + +/** + * Various utlity functions to help with the neceesary API calls + */ +export function isSuccessStatusCode(statusCode: number): boolean { + return statusCode >= 200 && statusCode < 300 +} + +export function isRetryableStatusCode(statusCode: number): boolean { + const retryableStatusCodes = [ + HttpCodes.BadGateway, + HttpCodes.ServiceUnavailable, + HttpCodes.GatewayTimeout + ] + return retryableStatusCodes.includes(statusCode) +} + +export function getContentRange( + start: number, + end: number, + total: number +): string { + // Format: `bytes start-end/filesize + // start and end are inclusive + // For a 200 byte chunk starting at byte 0: + // Content-Range: bytes 0-199/200 + return `bytes ${start}-${end}/${total}` +} + +export function getArtifactUrl(): string { + const runtimeUrl = process.env['ACTIONS_RUNTIME_URL'] + if (!runtimeUrl) { + throw new Error('Runtime url not found, unable to create artifact.') + } + const artifactUrl = `${runtimeUrl}_apis/pipelines/workflows/${getWorkFlowRunId()}/artifacts?api-version=${apiVersion}` + debug(`Artifact Url: ${artifactUrl}`) + return artifactUrl +} + +export function getRequestOptions(): IHeaders { + const requestOptions: IHeaders = { + Accept: createAcceptHeader('application/json', apiVersion) + } + return requestOptions +} + +export function createAcceptHeader(type: string, apiVersion: string): string { + return `${type};api-version=${apiVersion}` +} + +function getWorkFlowRunId(): string { + const workFlowrunId = process.env['GITHUB_RUN_ID'] || '' + if (!workFlowrunId) { + throw new Error('Unable to get workFlowRunId') + } + return workFlowrunId! +} + +/** + * Invalid characters that cannot be in the artifact name or an uploaded file. Will be rejected + * from the server if attempted to be sent over. These characters are not allowed due to limitations with certain + * file systems such as NTFS. To maitain platform-agnostic behavior, all characters that are not supported by an + * individual filesystem/platform will not be supported on all filesystems/platforms + */ +const invalidCharacters = ['\\', '/', '"', ':', '<', '>', '|', '*', '?'] + +/** + * Scans the name of the item being uploaded to make sure there are no illegal characters + */ +export function checkArtifactName(name: string) { + invalidCharacters.forEach(function(invalidChar) { + if (name.indexOf(invalidChar) > -1) { + throw new Error( + `Artifact name is not valid: ${name}. Contains character: "${invalidChar}". Invalid characters include: ${invalidCharacters.toString()}.` + ) + } + }) +} diff --git a/packages/artifact/tsconfig.json b/packages/artifact/tsconfig.json new file mode 100644 index 0000000000..a8b812a6ff --- /dev/null +++ b/packages/artifact/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "baseUrl": "./", + "outDir": "./lib", + "rootDir": "./src" + }, + "include": [ + "./src" + ] +} \ No newline at end of file From 776270f1c511328e1b03eafb6278eb0d72b845b6 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Mon, 13 Jan 2020 11:16:11 -0500 Subject: [PATCH 02/46] Alphabetize imports --- packages/artifact/src/artifact.ts | 8 ++++---- packages/artifact/src/search.ts | 4 ++-- packages/artifact/src/upload-artifact-http-client.ts | 10 +++++----- packages/artifact/src/utils.ts | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/artifact/src/artifact.ts b/packages/artifact/src/artifact.ts index e0609016dc..cef0c65d9b 100644 --- a/packages/artifact/src/artifact.ts +++ b/packages/artifact/src/artifact.ts @@ -1,13 +1,13 @@ -import {UploadInfo} from './upload-info' +import * as core from '@actions/core' +import {CreateArtifactResponse} from './contracts' +import {SearchResult, findFilesToUpload} from './search' import { createArtifactInFileContainer, uploadArtifactToFileContainer, patchArtifactSize } from './upload-artifact-http-client' +import {UploadInfo} from './upload-info' import {checkArtifactName} from './utils' -import * as core from '@actions/core' -import {CreateArtifactResponse} from './contracts' -import {SearchResult, findFilesToUpload} from './search' /** * Uploads an artifact diff --git a/packages/artifact/src/search.ts b/packages/artifact/src/search.ts index 3c15ae1d09..79c84fd07b 100644 --- a/packages/artifact/src/search.ts +++ b/packages/artifact/src/search.ts @@ -1,7 +1,7 @@ -import * as glob from '@actions/glob' -import {join, basename} from 'path' import {debug} from '@actions/core' +import * as glob from '@actions/glob' import {lstatSync} from 'fs' +import {join, basename} from 'path' export interface SearchResult { absoluteFilePath: string diff --git a/packages/artifact/src/upload-artifact-http-client.ts b/packages/artifact/src/upload-artifact-http-client.ts index 8b705e4a5f..76bbc5905f 100644 --- a/packages/artifact/src/upload-artifact-http-client.ts +++ b/packages/artifact/src/upload-artifact-http-client.ts @@ -1,13 +1,16 @@ import {debug} from '@actions/core' -import * as fs from 'fs' -import {HttpClientResponse, HttpClient} from '@actions/http-client/index' import {BearerCredentialHandler} from '@actions/http-client/auth' +import {HttpClientResponse, HttpClient} from '@actions/http-client/index' +import {IHttpClientResponse} from '@actions/http-client/interfaces' import { CreateArtifactResponse, CreateArtifactParameters, PatchArtifactSize, PatchArtifactSizeSuccessResponse } from './contracts' +import * as fs from 'fs' +import {SearchResult} from './search' +import {URL} from 'url' import { parseEnvNumber, getArtifactUrl, @@ -16,9 +19,6 @@ import { getRequestOptions, getContentRange } from './utils' -import {SearchResult} from './search' -import {IHttpClientResponse} from '@actions/http-client/interfaces' -import {URL} from 'url' const defaultChunkUploadConcurrency = 3 const defaultFileUploadConcurrency = 2 diff --git a/packages/artifact/src/utils.ts b/packages/artifact/src/utils.ts index b52fd16975..e43b478f99 100644 --- a/packages/artifact/src/utils.ts +++ b/packages/artifact/src/utils.ts @@ -1,5 +1,5 @@ -import {HttpCodes, HttpClient} from '@actions/http-client' import {debug} from '@actions/core' +import {HttpCodes} from '@actions/http-client' import {IHeaders} from '@actions/http-client/interfaces' const apiVersion: string = '6.0-preview' From f3ac7464d52a503de146af3c4f84d8ad12084ff8 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Mon, 13 Jan 2020 17:02:17 -0500 Subject: [PATCH 03/46] Fix majority of lint errors --- packages/artifact/src/artifact.ts | 21 +++---- packages/artifact/src/search.ts | 43 +++++++------- .../src/upload-artifact-http-client.ts | 59 +++++++++---------- packages/artifact/src/utils.ts | 16 ++--- 4 files changed, 68 insertions(+), 71 deletions(-) diff --git a/packages/artifact/src/artifact.ts b/packages/artifact/src/artifact.ts index cef0c65d9b..47f568de26 100644 --- a/packages/artifact/src/artifact.ts +++ b/packages/artifact/src/artifact.ts @@ -32,13 +32,14 @@ export async function uploadArtifact( } // Search for the items that will be uploaded - let filesToUpload: SearchResult[] = await findFilesToUpload(name, path) + const filesToUpload: SearchResult[] = await findFilesToUpload(name, path) + let reportedSize = -1 if (filesToUpload === undefined) { core.setFailed( `Unable to succesfully search fo files to upload with the provided path: ${path}` ) - } else if (filesToUpload.length == 0) { + } else if (filesToUpload.length === 0) { core.warning( `No files were found for the provided path: ${path}. No artifacts will be uploaded.` ) @@ -60,32 +61,28 @@ export async function uploadArtifact( * Step 2 of 3 * Upload each of the files that were found concurrently */ - let artfactSize: Promise = Promise.resolve( + const uploadingArtifact: Promise = Promise.resolve( uploadArtifactToFileContainer( - response.fileContainerResourceUrl!, + response.fileContainerResourceUrl, filesToUpload ) ) - artfactSize.then(async size => { + uploadingArtifact.then(async size => { console.log(`Size of what we just uploaded is ${size}`) /** * Step 3 of 3 * Update the size of the artifact to indicate we are done uploading */ await patchArtifactSize(size, name) - return { - artifactName: name, - artifactItems: filesToUpload.map(item => item.absoluteFilePath), - size: size - } as UploadInfo + reportedSize = size }) } return { artifactName: name, artifactItems: filesToUpload.map(item => item.absoluteFilePath), - size: -1 - } as UploadInfo + size: reportedSize + } } /* diff --git a/packages/artifact/src/search.ts b/packages/artifact/src/search.ts index 79c84fd07b..dc3c3798ef 100644 --- a/packages/artifact/src/search.ts +++ b/packages/artifact/src/search.ts @@ -18,30 +18,31 @@ export async function findFilesToUpload( artifactName: string, searchPath: string ): Promise { - let searchResults: SearchResult[] = [] - let itemsToUpload: string[] = [] - let options: glob.GlobOptions = { + const searchResults: SearchResult[] = [] + const itemsToUpload: string[] = [] + const options: glob.GlobOptions = { followSymbolicLinks: true, implicitDescendants: true, omitBrokenSymbolicLinks: true } - let globber = await glob.create(searchPath, options) + const globber = await glob.create(searchPath, options) - let rawSearchResults: string[] = await globber.glob() + const rawSearchResults: string[] = await globber.glob() /** * Directories will be rejected if attempted to be uploaded. This includes just empty * directories so filter any directories out from the raw search results */ - - rawSearchResults.forEach(function(item) { - if (!lstatSync(item).isDirectory()) { - itemsToUpload.push(item) + for (const searchResult of rawSearchResults) { + if (!lstatSync(searchResult).isDirectory()) { + itemsToUpload.push(searchResult) } else { - debug(`Removing ${item} from rawSearchResults because it is a directory`) + debug( + `Removing ${searchResult} from rawSearchResults because it is a directory` + ) } - }) + } - if (itemsToUpload.length == 0) { + if (itemsToUpload.length === 0) { return searchResults } console.log( @@ -53,7 +54,7 @@ export async function findFilesToUpload( * Only a single search pattern is being included so only 1 searchResult is expected. In the future if multiple search patterns are * simultaneously supported this will change */ - let searchPaths: string[] = await globber.getSearchPaths() + const searchPaths: string[] = globber.getSearchPaths() if (searchResults.length > 1) { console.log(searchResults) throw new Error('Only 1 search path should be returned') @@ -63,7 +64,7 @@ export async function findFilesToUpload( * Creates the path that the artifact will be uploaded with. The artifact name will always be the root directory so that * it can be distinguished from other artifacts that are uploaded to the same file Container/glob storage during a run */ - if (itemsToUpload.length == 1) { + if (itemsToUpload.length === 1) { // A single artifact will be uploaded, the upload path will always be in the form ${artifactName}\${singleArtifactName} searchResults.push({ absoluteFilePath: itemsToUpload[0], @@ -75,21 +76,21 @@ export async function findFilesToUpload( * The search path denotes the base path that was used to find the file. It will be removed from the absolute path and * the artifact name will be prepended to create the path used during upload */ - itemsToUpload.forEach(function(file) { - let uploadPath: string = file.replace(searchPaths[0], '') + for (const uploadItem of itemsToUpload) { + const uploadPath: string = uploadItem.replace(searchPaths[0], '') searchResults.push({ - absoluteFilePath: file, + absoluteFilePath: uploadItem, uploadFilePath: artifactName.concat(uploadPath) }) - }) + } } debug('SearchResult includes the following information:') - searchResults.forEach(function(item) { + for (const searchResult of searchResults) { debug( - `Absolute File Path: ${item.absoluteFilePath}\nUpload file path: ${item.uploadFilePath}` + `Absolute File Path: ${searchResult.absoluteFilePath}\nUpload file path: ${searchResult.uploadFilePath}` ) - }) + } return searchResults } diff --git a/packages/artifact/src/upload-artifact-http-client.ts b/packages/artifact/src/upload-artifact-http-client.ts index 76bbc5905f..571cc9b7a4 100644 --- a/packages/artifact/src/upload-artifact-http-client.ts +++ b/packages/artifact/src/upload-artifact-http-client.ts @@ -36,26 +36,25 @@ export async function createArtifactInFileContainer( const requestOptions = getRequestOptions() requestOptions['Content-Type'] = 'application/json' - let client: HttpClient = new HttpClient('actions/artifact', [ + const client: HttpClient = new HttpClient('actions/artifact', [ bearerCredentialHandler ]) - let parameters: CreateArtifactParameters = { + const parameters: CreateArtifactParameters = { Type: 'actions_storage', Name: artifactName } - let data: string = JSON.stringify(parameters, null, 2) - - const artifactUrl: string = getArtifactUrl() + const data: string = JSON.stringify(parameters, null, 2) const rawResponse: HttpClientResponse = await client.post( - artifactUrl, + getArtifactUrl(), data, requestOptions ) - let body: string = await rawResponse.readBody() - let response: CreateArtifactResponse = JSON.parse(body) + + const body: string = await rawResponse.readBody() + const response: CreateArtifactResponse = JSON.parse(body) console.log(response) - if (rawResponse.message.statusCode == 201 && response) { + if (rawResponse.message.statusCode === 201 && response) { return response } else { throw new Error( @@ -76,7 +75,7 @@ export async function uploadArtifactToFileContainer( ): Promise { const token = process.env['ACTIONS_RUNTIME_TOKEN'] || '' const bearerCredentialHandler = new BearerCredentialHandler(token) - let client: HttpClient = new HttpClient('actions/artifact', [ + const client: HttpClient = new HttpClient('actions/artifact', [ bearerCredentialHandler ]) @@ -92,44 +91,44 @@ export async function uploadArtifactToFileContainer( `File Concurrency: ${FILE_CONCURRENCY}, Chunk Concurrency: ${CHUNK_CONCURRENCY} and Chunk Size: ${MAX_CHUNK_SIZE}` ) - let parameters: UploadFileParameters[] = [] + const parameters: UploadFileParameters[] = [] // Prepare the necessary parameters to upload all the files - filesToUpload.forEach(function(item) { - let resourceUrl = new URL(uploadUrl) + for (const file of filesToUpload) { + const resourceUrl = new URL(uploadUrl) resourceUrl.searchParams.append( 'scope', '00000000-0000-0000-0000-000000000000' ) - resourceUrl.searchParams.append('itemPath', item.uploadFilePath) + resourceUrl.searchParams.append('itemPath', file.uploadFilePath) parameters.push({ - file: item.absoluteFilePath, + file: file.absoluteFilePath, resourceUrl: resourceUrl.toString(), restClient: client, concurrency: CHUNK_CONCURRENCY, maxChunkSize: MAX_CHUNK_SIZE }) - }) + } const parallelUploads = [...new Array(FILE_CONCURRENCY).keys()] + const fileSizes: number[] = [] let uploadedFiles = 0 - let fileSizes: number[] = [] // Only allow a certain amount of files to be uploaded at once, this is done to reduce errors if // trying to upload everything at once await Promise.all( parallelUploads.map(async () => { while (uploadedFiles < filesToUpload.length) { - let currentFileParameters = parameters[uploadedFiles] + const currentFileParameters = parameters[uploadedFiles] uploadedFiles += 1 fileSizes.push(await uploadFileAsync(currentFileParameters)) } }) ) - let sum: number = 0 - for (var i = 0; i < fileSizes.length; i++) { - sum += fileSizes[i] + let sum = 0 + for (const fileSize of fileSizes) { + sum += fileSize } console.log(`Total size of all the files uploaded ${sum}`) return sum @@ -258,32 +257,32 @@ export async function patchArtifactSize( const requestOptions = getRequestOptions() requestOptions['Content-Type'] = 'application/json' - let resourceUrl = new URL(getArtifactUrl()) + const resourceUrl = new URL(getArtifactUrl()) resourceUrl.searchParams.append('artifactName', artifactName) - let parameters: PatchArtifactSize = {Size: size} - let data: string = JSON.stringify(parameters, null, 2) + const parameters: PatchArtifactSize = {Size: size} + const data: string = JSON.stringify(parameters, null, 2) const token = process.env['ACTIONS_RUNTIME_TOKEN'] || '' const bearerCredentialHandler = new BearerCredentialHandler(token) - let client: HttpClient = new HttpClient('actions/artifact', [ + const client: HttpClient = new HttpClient('actions/artifact', [ bearerCredentialHandler ]) console.log(`URL is ${resourceUrl.toString()}`) - let rawResponse: HttpClientResponse = await client.patch( + const rawResponse: HttpClientResponse = await client.patch( resourceUrl.toString(), data, requestOptions ) - let body: string = await rawResponse.readBody() + const body: string = await rawResponse.readBody() - if (rawResponse.message.statusCode == 200) { - let successResponse: PatchArtifactSizeSuccessResponse = JSON.parse(body) + if (rawResponse.message.statusCode === 200) { + const successResponse: PatchArtifactSizeSuccessResponse = JSON.parse(body) console.log('Artifact size was succesfully updated!') console.log(successResponse) - } else if (rawResponse.message.statusCode == 404) { + } else if (rawResponse.message.statusCode === 404) { throw new Error(`An Artifact with the name ${artifactName} was not found`) } else { console.log(body) diff --git a/packages/artifact/src/utils.ts b/packages/artifact/src/utils.ts index e43b478f99..698a919121 100644 --- a/packages/artifact/src/utils.ts +++ b/packages/artifact/src/utils.ts @@ -2,7 +2,7 @@ import {debug} from '@actions/core' import {HttpCodes} from '@actions/http-client' import {IHeaders} from '@actions/http-client/interfaces' -const apiVersion: string = '6.0-preview' +const apiVersion = '6.0-preview' /** * Parses a env variable that is a number @@ -55,12 +55,12 @@ export function getArtifactUrl(): string { export function getRequestOptions(): IHeaders { const requestOptions: IHeaders = { - Accept: createAcceptHeader('application/json', apiVersion) + Accept: createAcceptHeader('application/json') } return requestOptions } -export function createAcceptHeader(type: string, apiVersion: string): string { +export function createAcceptHeader(type: string): string { return `${type};api-version=${apiVersion}` } @@ -69,7 +69,7 @@ function getWorkFlowRunId(): string { if (!workFlowrunId) { throw new Error('Unable to get workFlowRunId') } - return workFlowrunId! + return workFlowrunId } /** @@ -83,12 +83,12 @@ const invalidCharacters = ['\\', '/', '"', ':', '<', '>', '|', '*', '?'] /** * Scans the name of the item being uploaded to make sure there are no illegal characters */ -export function checkArtifactName(name: string) { - invalidCharacters.forEach(function(invalidChar) { - if (name.indexOf(invalidChar) > -1) { +export function checkArtifactName(name: string): void { + for (const invalidChar of invalidCharacters) { + if (name.includes(invalidChar)) { throw new Error( `Artifact name is not valid: ${name}. Contains character: "${invalidChar}". Invalid characters include: ${invalidCharacters.toString()}.` ) } - }) + } } From bd38a3fe6eba19e7af5a6262d0d713d8df9db369 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Tue, 14 Jan 2020 10:29:23 -0500 Subject: [PATCH 04/46] Fix all lint errors --- packages/artifact/src/artifact.ts | 2 ++ packages/artifact/src/search.ts | 3 +++ packages/artifact/src/upload-artifact-http-client.ts | 12 +++++++++++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/packages/artifact/src/artifact.ts b/packages/artifact/src/artifact.ts index 47f568de26..7858355282 100644 --- a/packages/artifact/src/artifact.ts +++ b/packages/artifact/src/artifact.ts @@ -52,6 +52,7 @@ export async function uploadArtifact( name ) if (!response.fileContainerResourceUrl) { + // eslint-disable-next-line no-console console.log(response) throw new Error('Unable to get fileContainerResourceUrl') } @@ -68,6 +69,7 @@ export async function uploadArtifact( ) ) uploadingArtifact.then(async size => { + // eslint-disable-next-line no-console console.log(`Size of what we just uploaded is ${size}`) /** * Step 3 of 3 diff --git a/packages/artifact/src/search.ts b/packages/artifact/src/search.ts index dc3c3798ef..de3f38e998 100644 --- a/packages/artifact/src/search.ts +++ b/packages/artifact/src/search.ts @@ -45,9 +45,11 @@ export async function findFilesToUpload( if (itemsToUpload.length === 0) { return searchResults } + // eslint-disable-next-line no-console console.log( `Found the following ${itemsToUpload.length} items that will be uploaded as part of the artifact:` ) + // eslint-disable-next-line no-console console.log(itemsToUpload) /** @@ -56,6 +58,7 @@ export async function findFilesToUpload( */ const searchPaths: string[] = globber.getSearchPaths() if (searchResults.length > 1) { + // eslint-disable-next-line no-console console.log(searchResults) throw new Error('Only 1 search path should be returned') } diff --git a/packages/artifact/src/upload-artifact-http-client.ts b/packages/artifact/src/upload-artifact-http-client.ts index 571cc9b7a4..ec8f15f481 100644 --- a/packages/artifact/src/upload-artifact-http-client.ts +++ b/packages/artifact/src/upload-artifact-http-client.ts @@ -52,6 +52,7 @@ export async function createArtifactInFileContainer( const body: string = await rawResponse.readBody() const response: CreateArtifactResponse = JSON.parse(body) + // eslint-disable-next-line no-console console.log(response) if (rawResponse.message.statusCode === 201 && response) { @@ -130,6 +131,7 @@ export async function uploadArtifactToFileContainer( for (const fileSize of fileSizes) { sum += fileSize } + // eslint-disable-next-line no-console console.log(`Total size of all the files uploaded ${sum}`) return sum } @@ -194,6 +196,7 @@ async function uploadChunk( end: number, totalSize: number ): Promise { + // eslint-disable-next-line no-console console.log( `Uploading chunk of size ${end - start + @@ -216,6 +219,7 @@ async function uploadChunk( const response = await uploadChunkRequest() if (!response.message.statusCode) { + // eslint-disable-next-line no-console console.log(response) throw new Error('No Status Code returned with response') } @@ -227,14 +231,17 @@ async function uploadChunk( return } if (isRetryableStatusCode(response.message.statusCode)) { + // eslint-disable-next-line no-console console.log( `Received ${response.message.statusCode}, will retry chunk at offset ${start} after 10 seconds.` ) await new Promise(resolve => setTimeout(resolve, 10000)) + // eslint-disable-next-line no-console console.log(`Retrying chunk at offset ${start}`) const retryResponse = await uploadChunkRequest() if (!retryResponse.message.statusCode) { + // eslint-disable-next-line no-console console.log(retryResponse) throw new Error('No Status Code returne with response') } @@ -268,7 +275,7 @@ export async function patchArtifactSize( const client: HttpClient = new HttpClient('actions/artifact', [ bearerCredentialHandler ]) - + // eslint-disable-next-line no-console console.log(`URL is ${resourceUrl.toString()}`) const rawResponse: HttpClientResponse = await client.patch( @@ -280,11 +287,14 @@ export async function patchArtifactSize( if (rawResponse.message.statusCode === 200) { const successResponse: PatchArtifactSizeSuccessResponse = JSON.parse(body) + // eslint-disable-next-line no-console console.log('Artifact size was succesfully updated!') + // eslint-disable-next-line no-console console.log(successResponse) } else if (rawResponse.message.statusCode === 404) { throw new Error(`An Artifact with the name ${artifactName} was not found`) } else { + // eslint-disable-next-line no-console console.log(body) throw new Error('Unable to update the artifact size') } From b2d67bbb614ffeb9e0ee539bb5661298139c40b9 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Tue, 14 Jan 2020 18:47:22 -0500 Subject: [PATCH 05/46] Various fixes due to PR feedback --- packages/artifact/src/artifact.ts | 23 ++++++++----------- packages/artifact/src/contracts.ts | 14 +++++------ .../src/upload-artifact-http-client.ts | 14 +++++------ packages/artifact/src/utils.ts | 5 +++- 4 files changed, 27 insertions(+), 29 deletions(-) diff --git a/packages/artifact/src/artifact.ts b/packages/artifact/src/artifact.ts index 7858355282..997436eba5 100644 --- a/packages/artifact/src/artifact.ts +++ b/packages/artifact/src/artifact.ts @@ -1,5 +1,4 @@ import * as core from '@actions/core' -import {CreateArtifactResponse} from './contracts' import {SearchResult, findFilesToUpload} from './search' import { createArtifactInFileContainer, @@ -20,11 +19,6 @@ export async function uploadArtifact( name: string, path: string ): Promise { - // Check that the required inputs are valid - if (!name) { - throw new Error('Artifact name must be provided') - } - checkArtifactName(name) if (!path) { @@ -48,15 +42,14 @@ export async function uploadArtifact( * Step 1 of 3 * Create an entry for the artifact in the file container */ - const response: CreateArtifactResponse = await createArtifactInFileContainer( - name - ) + const response = await createArtifactInFileContainer(name) if (!response.fileContainerResourceUrl) { - // eslint-disable-next-line no-console - console.log(response) - throw new Error('Unable to get fileContainerResourceUrl') + core.debug(response.toString()) + throw new Error( + 'No URL provided by the Artifact Service to upload an artifact to' + ) } - core.debug(`We will be uploading to: ${response.fileContainerResourceUrl}`) + core.debug(`Upload Resource URL: ${response.fileContainerResourceUrl}`) /** * Step 2 of 3 @@ -70,7 +63,9 @@ export async function uploadArtifact( ) uploadingArtifact.then(async size => { // eslint-disable-next-line no-console - console.log(`Size of what we just uploaded is ${size}`) + console.log( + `All files for artifact ${name} have finished uploading. Reported upload size is ${size} bytes` + ) /** * Step 3 of 3 * Update the size of the artifact to indicate we are done uploading diff --git a/packages/artifact/src/contracts.ts b/packages/artifact/src/contracts.ts index b0bbaa9434..dc97cac735 100644 --- a/packages/artifact/src/contracts.ts +++ b/packages/artifact/src/contracts.ts @@ -1,11 +1,11 @@ export interface CreateArtifactResponse { - containerId?: string - size?: number - signedContent?: string - fileContainerResourceUrl?: string - type?: string - name?: string - url?: string + containerId: string + size: number + signedContent: string + fileContainerResourceUrl: string + type: string + name: string + url: string } export interface CreateArtifactParameters { diff --git a/packages/artifact/src/upload-artifact-http-client.ts b/packages/artifact/src/upload-artifact-http-client.ts index ec8f15f481..0fcea6f510 100644 --- a/packages/artifact/src/upload-artifact-http-client.ts +++ b/packages/artifact/src/upload-artifact-http-client.ts @@ -127,10 +127,8 @@ export async function uploadArtifactToFileContainer( }) ) - let sum = 0 - for (const fileSize of fileSizes) { - sum += fileSize - } + // Sum up all the files that were uploaded + const sum = fileSizes.reduce((acc, val) => acc + val) // eslint-disable-next-line no-console console.log(`Total size of all the files uploaded ${sum}`) return sum @@ -175,7 +173,7 @@ async function uploadFileAsync( } }) ) - return fs.statSync(parameters.file).size + return fileSize } /** @@ -288,7 +286,9 @@ export async function patchArtifactSize( if (rawResponse.message.statusCode === 200) { const successResponse: PatchArtifactSizeSuccessResponse = JSON.parse(body) // eslint-disable-next-line no-console - console.log('Artifact size was succesfully updated!') + console.log( + `Artifact ${artifactName} uploaded successfully, total size ${size}` + ) // eslint-disable-next-line no-console console.log(successResponse) } else if (rawResponse.message.statusCode === 404) { @@ -296,7 +296,7 @@ export async function patchArtifactSize( } else { // eslint-disable-next-line no-console console.log(body) - throw new Error('Unable to update the artifact size') + throw new Error(`Unable to finish uploading artifact ${artifactName}`) } } diff --git a/packages/artifact/src/utils.ts b/packages/artifact/src/utils.ts index 698a919121..41d28df3f8 100644 --- a/packages/artifact/src/utils.ts +++ b/packages/artifact/src/utils.ts @@ -75,7 +75,7 @@ function getWorkFlowRunId(): string { /** * Invalid characters that cannot be in the artifact name or an uploaded file. Will be rejected * from the server if attempted to be sent over. These characters are not allowed due to limitations with certain - * file systems such as NTFS. To maitain platform-agnostic behavior, all characters that are not supported by an + * file systems such as NTFS. To maintain platform-agnostic behavior, all characters that are not supported by an * individual filesystem/platform will not be supported on all filesystems/platforms */ const invalidCharacters = ['\\', '/', '"', ':', '<', '>', '|', '*', '?'] @@ -84,6 +84,9 @@ const invalidCharacters = ['\\', '/', '"', ':', '<', '>', '|', '*', '?'] * Scans the name of the item being uploaded to make sure there are no illegal characters */ export function checkArtifactName(name: string): void { + if (!name) { + throw new Error(`Artifact name: ${name}, is incorrectly provided`) + } for (const invalidChar of invalidCharacters) { if (name.includes(invalidChar)) { throw new Error( From 029f2a43f1b100c95da758162a2bafa8ad2b2e03 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Tue, 14 Jan 2020 20:47:06 -0500 Subject: [PATCH 06/46] Add tests for search --- .../__tests__/internal-search.test.ts | 211 ++++++++++++++++++ packages/artifact/src/search.ts | 2 +- 2 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 packages/artifact/__tests__/internal-search.test.ts diff --git a/packages/artifact/__tests__/internal-search.test.ts b/packages/artifact/__tests__/internal-search.test.ts new file mode 100644 index 0000000000..4c30d8f310 --- /dev/null +++ b/packages/artifact/__tests__/internal-search.test.ts @@ -0,0 +1,211 @@ +import {SearchResult, findFilesToUpload} from '../src/search' +import * as path from 'path' +import * as io from '../../io/src/io' +import {promises as fs} from 'fs' + +describe('search', () => { + // Remove temp directory after each test + afterEach(async () => { + await io.rmRF(getTestTemp()) + }) + + /** + * Creates a single item in /single-file-artifact/folder-a/folder-b/folder-b/file-under-c.txt + * Expected to find that item with full file path provided + * Only 1 item is expected to be found so the uploadFilePath is expected to be {artifactName}/file-under-c.txt + */ + it('Single file search - full path', async () => { + const artifactName = "my-artifact" + const root = path.join(getTestTemp(), 'single-file-artifact') + await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { + recursive: true + }) + const itemPath = path.join(root, 'folder-a', 'folder-b', 'folder-c', 'file-under-c.txt') + await fs.writeFile( + itemPath, + 'sample file under folder c' + ) + + const exepectedUploadFilePath = path.join(artifactName,'file-under-c.txt') + const searchResult = await findFilesToUpload(artifactName, itemPath) + expect(searchResult.length).toEqual(1) + expect(searchResult[0].uploadFilePath).toEqual(exepectedUploadFilePath) + expect(searchResult[0].absoluteFilePath).toEqual(itemPath) + }) + + /** + * Creates a single item in /single-file-artifact/folder-a/folder-b/folder-b/file-under-c.txt + * Expected to find that one item with a provided wildcard pattern + * Only 1 item is expected to be found so the uploadFilePath is expected to be {artifactName}/file-under-c.txt + */ + it('Single file search - wildcard pattern', async () => { + const artifactName = "my-artifact" + const root = path.join(getTestTemp(), 'single-file-artifact') + await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { + recursive: true + }) + const itemPath = path.join(root, 'folder-a', 'folder-b', 'folder-c', 'item1.txt') + await fs.writeFile( + itemPath, + 'sample file under folder c' + ) + const searchPath = '**/*m1.txt' + const exepectedUploadFilePath = path.join(artifactName,'item1.txt') + const searchResult = await findFilesToUpload(artifactName, searchPath) + expect(searchResult.length).toEqual(1) + expect(searchResult[0].uploadFilePath).toEqual(exepectedUploadFilePath) + expect(searchResult[0].absoluteFilePath).toEqual(itemPath) + }) + + /** + * Creates a directory with multiple files and subdirectories, no empty directories + * All items are expected to be found + */ + it('Directory search - no empty directories', async () => { + const artifactName = "my-artifact" + const root = path.join(getTestTemp(), 'single-file-artifact') + await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-d'), { + recursive: true + }) + const item1Path = path.join(root, 'folder-a', 'folder-b', 'folder-c', 'item1.txt') + const item2Path = path.join(root, 'folder-d', 'item2.txt') + const item3Path = path.join(root, 'folder-d', 'item3.txt') + const item4Path = path.join(root, 'folder-d', 'item4.txt') + const item5Path = path.join(root, 'item5.txt') + await fs.writeFile(item1Path, 'item1 file') + await fs.writeFile(item2Path, 'item2 file') + await fs.writeFile(item3Path, 'item3 file') + await fs.writeFile(item4Path, 'item4 file') + await fs.writeFile(item5Path, 'item5 file') + /* + Directory structure of files that were created: + root/ + folder-a/ + folder-b/ + folder-c/ + item1.txt + folder-d/ + item2.txt + item3.txt + item4.txt + item5.txt + */ + const searchResult = await findFilesToUpload(artifactName, root) + expect(searchResult.length).toEqual(5) + + const absolutePaths = searchResult.map(item => item.absoluteFilePath) + expect(absolutePaths.includes(item1Path)).toEqual(true); + expect(absolutePaths.includes(item2Path)).toEqual(true); + expect(absolutePaths.includes(item3Path)).toEqual(true); + expect(absolutePaths.includes(item4Path)).toEqual(true); + expect(absolutePaths.includes(item5Path)).toEqual(true); + + for (const result of searchResult){ + if(result.absoluteFilePath === item1Path){ + expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-a', 'folder-b', 'folder-c', 'item1.txt')); + } + if(result.absoluteFilePath === item2Path){ + expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-d', 'item2.txt')); + } + if(result.absoluteFilePath === item3Path){ + expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-d', 'item3.txt')); + } + if(result.absoluteFilePath === item4Path){ + expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-d', 'item4.txt')); + } + if(result.absoluteFilePath === item5Path){ + expect(result.uploadFilePath).toEqual(path.join(artifactName,'item5.txt')); + } + } + }) + + /** + * Creates a directory with multiple files and subdirectories, includes some empty directories + * All items are expected to be found + */ + it('Directory search - with empty directories', async () => { + const artifactName = "my-artifact" + const root = path.join(getTestTemp(), 'single-file-artifact') + await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-e'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-d'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-f'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-g'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-h', 'folder-i'), { + recursive: true + }) + const item1Path = path.join(root, 'folder-a', 'folder-b', 'folder-c', 'item1.txt') + const item2Path = path.join(root, 'folder-d', 'item2.txt') + const item3Path = path.join(root, 'folder-d', 'item3.txt') + const item4Path = path.join(root, 'folder-d', 'item4.txt') + const item5Path = path.join(root, 'item5.txt') + await fs.writeFile(item1Path, 'item1 file') + await fs.writeFile(item2Path, 'item2 file') + await fs.writeFile(item3Path, 'item3 file') + await fs.writeFile(item4Path, 'item4 file') + await fs.writeFile(item5Path, 'item5 file') + /* + Directory structure of files that were created: + root/ + folder-a/ + folder-b/ + folder-c/ + item1.txt + folder-e/ + folder-d/ + item2.txt + item3.txt + item4.txt + folder-f/ + folder-g/ + folder-h/ + folder-i/ + item5.txt + */ + const searchResult = await findFilesToUpload(artifactName, root) + expect(searchResult.length).toEqual(5) + + const absolutePaths = searchResult.map(item => item.absoluteFilePath) + expect(absolutePaths.includes(item1Path)).toEqual(true); + expect(absolutePaths.includes(item2Path)).toEqual(true); + expect(absolutePaths.includes(item3Path)).toEqual(true); + expect(absolutePaths.includes(item4Path)).toEqual(true); + expect(absolutePaths.includes(item5Path)).toEqual(true); + + for (const result of searchResult){ + if(result.absoluteFilePath === item1Path){ + expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-a', 'folder-b', 'folder-c', 'item1.txt')); + } + if(result.absoluteFilePath === item2Path){ + expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-d', 'item2.txt')); + } + if(result.absoluteFilePath === item3Path){ + expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-d', 'item3.txt')); + } + if(result.absoluteFilePath === item4Path){ + expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-d', 'item4.txt')); + } + if(result.absoluteFilePath === item5Path){ + expect(result.uploadFilePath).toEqual(path.join(artifactName,'item5.txt')); + } + } + }) + + function getTestTemp(): string { + return path.join(__dirname, '_temp', 'artifact') + } + +}) \ No newline at end of file diff --git a/packages/artifact/src/search.ts b/packages/artifact/src/search.ts index de3f38e998..c0a37ca1ba 100644 --- a/packages/artifact/src/search.ts +++ b/packages/artifact/src/search.ts @@ -71,7 +71,7 @@ export async function findFilesToUpload( // A single artifact will be uploaded, the upload path will always be in the form ${artifactName}\${singleArtifactName} searchResults.push({ absoluteFilePath: itemsToUpload[0], - uploadFilePath: join(artifactName, basename(searchPaths[0])) + uploadFilePath: join(artifactName, basename(itemsToUpload[0])) }) } else { /** From 4fda76641316e5cce02d55e22c9d940c8a24dd40 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Tue, 14 Jan 2020 20:49:20 -0500 Subject: [PATCH 07/46] Rename test file --- .../__tests__/{internal-search.test.ts => search.test.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/artifact/__tests__/{internal-search.test.ts => search.test.ts} (100%) diff --git a/packages/artifact/__tests__/internal-search.test.ts b/packages/artifact/__tests__/search.test.ts similarity index 100% rename from packages/artifact/__tests__/internal-search.test.ts rename to packages/artifact/__tests__/search.test.ts From e025ef007f5d58036be816ebfb56f56059b0270c Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Tue, 14 Jan 2020 21:10:40 -0500 Subject: [PATCH 08/46] Test improvements --- packages/artifact/__tests__/search.test.ts | 103 ++++++++++++++++++++- packages/artifact/package.json | 2 +- 2 files changed, 102 insertions(+), 3 deletions(-) diff --git a/packages/artifact/__tests__/search.test.ts b/packages/artifact/__tests__/search.test.ts index 4c30d8f310..d1007de63e 100644 --- a/packages/artifact/__tests__/search.test.ts +++ b/packages/artifact/__tests__/search.test.ts @@ -49,7 +49,7 @@ describe('search', () => { itemPath, 'sample file under folder c' ) - const searchPath = '**/*m1.txt' + const searchPath = path.join(root,'**/*m1.txt') const exepectedUploadFilePath = path.join(artifactName,'item1.txt') const searchResult = await findFilesToUpload(artifactName, searchPath) expect(searchResult.length).toEqual(1) @@ -204,8 +204,107 @@ describe('search', () => { } }) + /** + * Creates a directory with multiple files and subdirectories, includes some empty directories and misc files + * Only files corresponding to the good* pattern should be found + */ + it('Wildcard search for mulitple files', async () => { + const artifactName = "my-artifact" + const root = path.join(getTestTemp(), 'single-file-artifact') + await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-e'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-d'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-f'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-g'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-h', 'folder-i'), { + recursive: true + }) + const goodItem1Path = path.join(root, 'folder-a', 'folder-b', 'folder-c', 'good-item1.txt') + const goodItem2Path = path.join(root, 'folder-d', 'good-item2.txt') + const goodItem3Path = path.join(root, 'folder-d', 'good-item3.txt') + const goodItem4Path = path.join(root, 'folder-d', 'good-item4.txt') + const goodItem5Path = path.join(root, 'good-item5.txt') + await fs.writeFile(goodItem1Path, 'good item1 file') + await fs.writeFile(goodItem2Path, 'good item2 file') + await fs.writeFile(goodItem3Path, 'good item3 file') + await fs.writeFile(goodItem4Path, 'good item4 file') + await fs.writeFile(goodItem5Path, 'good item5 file') + + const badItem1Path = path.join(root, 'folder-a', 'folder-b', 'folder-c', 'bad-item1.txt') + const badItem2Path = path.join(root, 'folder-d', 'bad-item2.txt') + const badItem3Path = path.join(root, 'folder-f', 'bad-item3.txt') + const badItem4Path = path.join(root, 'folder-h', 'folder-i', 'bad-item4.txt') + const badItem5Path = path.join(root, 'folder-h', 'folder-i', 'bad-item5.txt') + await fs.writeFile(badItem1Path, 'bad item1 file') + await fs.writeFile(badItem2Path, 'bad item2 file') + await fs.writeFile(badItem3Path, 'bad item3 file') + await fs.writeFile(badItem4Path, 'bad item4 file') + await fs.writeFile(badItem5Path, 'bad item5 file') + + /* + Directory structure of files that were created: + root/ + folder-a/ + folder-b/ + folder-c/ + good-item1.txt + bad-item1.txt + folder-e/ + folder-d/ + good-item2.txt + good-item3.txt + good-item4.txt + bad-item2.txt + folder-f/ + bad-item3.txt + folder-g/ + folder-h/ + folder-i/ + bad-item4.txt + bad-item5.txt + good-item5.txt + */ + const searchPath = path.join(root, '**/good*') + const searchResult = await findFilesToUpload(artifactName, searchPath) + expect(searchResult.length).toEqual(5) + + const absolutePaths = searchResult.map(item => item.absoluteFilePath) + expect(absolutePaths.includes(goodItem1Path)).toEqual(true); + expect(absolutePaths.includes(goodItem2Path)).toEqual(true); + expect(absolutePaths.includes(goodItem3Path)).toEqual(true); + expect(absolutePaths.includes(goodItem4Path)).toEqual(true); + expect(absolutePaths.includes(goodItem5Path)).toEqual(true); + + for (const result of searchResult){ + if(result.absoluteFilePath === goodItem1Path){ + expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-a', 'folder-b', 'folder-c', 'good-item1.txt')); + } + if(result.absoluteFilePath === goodItem2Path){ + expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-d', 'good-item2.txt')); + } + if(result.absoluteFilePath === goodItem3Path){ + expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-d', 'good-item3.txt')); + } + if(result.absoluteFilePath === goodItem4Path){ + expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-d', 'good-item4.txt')); + } + if(result.absoluteFilePath === goodItem5Path){ + expect(result.uploadFilePath).toEqual(path.join(artifactName, 'good-item5.txt')); + } + } + }) + function getTestTemp(): string { return path.join(__dirname, '_temp', 'artifact') } - }) \ No newline at end of file diff --git a/packages/artifact/package.json b/packages/artifact/package.json index b2ed07a5dd..765d543bf8 100644 --- a/packages/artifact/package.json +++ b/packages/artifact/package.json @@ -29,7 +29,7 @@ }, "scripts": { "audit-moderate": "npm install && npm audit --audit-level=moderate", - "test": "echo \"Error: no test specified\" && exit 1", + "test": "echo \"Error: run tests from root\" && exit 1", "tsc": "tsc" }, "bugs": { From 11c006c7e2e2224456305251187757d6f5e419b3 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Tue, 14 Jan 2020 21:16:38 -0500 Subject: [PATCH 09/46] Misc formatting fixes --- packages/artifact/__tests__/search.test.ts | 566 ++++++++++++--------- 1 file changed, 327 insertions(+), 239 deletions(-) diff --git a/packages/artifact/__tests__/search.test.ts b/packages/artifact/__tests__/search.test.ts index d1007de63e..84c3be9610 100644 --- a/packages/artifact/__tests__/search.test.ts +++ b/packages/artifact/__tests__/search.test.ts @@ -1,86 +1,98 @@ -import {SearchResult, findFilesToUpload} from '../src/search' +import {findFilesToUpload} from '../src/search' import * as path from 'path' import * as io from '../../io/src/io' import {promises as fs} from 'fs' describe('search', () => { - // Remove temp directory after each test - afterEach(async () => { - await io.rmRF(getTestTemp()) + // Remove temp directory after each test + afterEach(async () => { + await io.rmRF(getTestTemp()) + }) + + /** + * Creates a single item in /single-file-artifact/folder-a/folder-b/folder-b/file-under-c.txt + * Expected to find that item with full file path provided + * Only 1 item is expected to be found so the uploadFilePath is expected to be {artifactName}/file-under-c.txt + */ + it('Single file search - full path', async () => { + const artifactName = 'my-artifact' + const root = path.join(getTestTemp(), 'single-file-artifact') + await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { + recursive: true }) + const itemPath = path.join( + root, + 'folder-a', + 'folder-b', + 'folder-c', + 'file-under-c.txt' + ) + await fs.writeFile(itemPath, 'sample file under folder c') - /** - * Creates a single item in /single-file-artifact/folder-a/folder-b/folder-b/file-under-c.txt - * Expected to find that item with full file path provided - * Only 1 item is expected to be found so the uploadFilePath is expected to be {artifactName}/file-under-c.txt - */ - it('Single file search - full path', async () => { - const artifactName = "my-artifact" - const root = path.join(getTestTemp(), 'single-file-artifact') - await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { - recursive: true - }) - const itemPath = path.join(root, 'folder-a', 'folder-b', 'folder-c', 'file-under-c.txt') - await fs.writeFile( - itemPath, - 'sample file under folder c' - ) + const exepectedUploadFilePath = path.join(artifactName, 'file-under-c.txt') + const searchResult = await findFilesToUpload(artifactName, itemPath) + expect(searchResult.length).toEqual(1) + expect(searchResult[0].uploadFilePath).toEqual(exepectedUploadFilePath) + expect(searchResult[0].absoluteFilePath).toEqual(itemPath) + }) - const exepectedUploadFilePath = path.join(artifactName,'file-under-c.txt') - const searchResult = await findFilesToUpload(artifactName, itemPath) - expect(searchResult.length).toEqual(1) - expect(searchResult[0].uploadFilePath).toEqual(exepectedUploadFilePath) - expect(searchResult[0].absoluteFilePath).toEqual(itemPath) + /** + * Creates a single item in /single-file-artifact/folder-a/folder-b/folder-b/file-under-c.txt + * Expected to find that one item with a provided wildcard pattern + * Only 1 item is expected to be found so the uploadFilePath is expected to be {artifactName}/file-under-c.txt + */ + it('Single file search - wildcard pattern', async () => { + const artifactName = 'my-artifact' + const root = path.join(getTestTemp(), 'single-file-artifact') + await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { + recursive: true }) + const itemPath = path.join( + root, + 'folder-a', + 'folder-b', + 'folder-c', + 'item1.txt' + ) + await fs.writeFile(itemPath, 'sample file under folder c') + const searchPath = path.join(root, '**/*m1.txt') + const exepectedUploadFilePath = path.join(artifactName, 'item1.txt') + const searchResult = await findFilesToUpload(artifactName, searchPath) + expect(searchResult.length).toEqual(1) + expect(searchResult[0].uploadFilePath).toEqual(exepectedUploadFilePath) + expect(searchResult[0].absoluteFilePath).toEqual(itemPath) + }) - /** - * Creates a single item in /single-file-artifact/folder-a/folder-b/folder-b/file-under-c.txt - * Expected to find that one item with a provided wildcard pattern - * Only 1 item is expected to be found so the uploadFilePath is expected to be {artifactName}/file-under-c.txt - */ - it('Single file search - wildcard pattern', async () => { - const artifactName = "my-artifact" - const root = path.join(getTestTemp(), 'single-file-artifact') - await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { - recursive: true - }) - const itemPath = path.join(root, 'folder-a', 'folder-b', 'folder-c', 'item1.txt') - await fs.writeFile( - itemPath, - 'sample file under folder c' - ) - const searchPath = path.join(root,'**/*m1.txt') - const exepectedUploadFilePath = path.join(artifactName,'item1.txt') - const searchResult = await findFilesToUpload(artifactName, searchPath) - expect(searchResult.length).toEqual(1) - expect(searchResult[0].uploadFilePath).toEqual(exepectedUploadFilePath) - expect(searchResult[0].absoluteFilePath).toEqual(itemPath) + /** + * Creates a directory with multiple files and subdirectories, no empty directories + * All items are expected to be found + */ + it('Directory search - no empty directories', async () => { + const artifactName = 'my-artifact' + const root = path.join(getTestTemp(), 'single-file-artifact') + await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { + recursive: true }) - - /** - * Creates a directory with multiple files and subdirectories, no empty directories - * All items are expected to be found - */ - it('Directory search - no empty directories', async () => { - const artifactName = "my-artifact" - const root = path.join(getTestTemp(), 'single-file-artifact') - await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-d'), { - recursive: true - }) - const item1Path = path.join(root, 'folder-a', 'folder-b', 'folder-c', 'item1.txt') - const item2Path = path.join(root, 'folder-d', 'item2.txt') - const item3Path = path.join(root, 'folder-d', 'item3.txt') - const item4Path = path.join(root, 'folder-d', 'item4.txt') - const item5Path = path.join(root, 'item5.txt') - await fs.writeFile(item1Path, 'item1 file') - await fs.writeFile(item2Path, 'item2 file') - await fs.writeFile(item3Path, 'item3 file') - await fs.writeFile(item4Path, 'item4 file') - await fs.writeFile(item5Path, 'item5 file') - /* + await fs.mkdir(path.join(root, 'folder-d'), { + recursive: true + }) + const item1Path = path.join( + root, + 'folder-a', + 'folder-b', + 'folder-c', + 'item1.txt' + ) + const item2Path = path.join(root, 'folder-d', 'item2.txt') + const item3Path = path.join(root, 'folder-d', 'item3.txt') + const item4Path = path.join(root, 'folder-d', 'item4.txt') + const item5Path = path.join(root, 'item5.txt') + await fs.writeFile(item1Path, 'item1 file') + await fs.writeFile(item2Path, 'item2 file') + await fs.writeFile(item3Path, 'item3 file') + await fs.writeFile(item4Path, 'item4 file') + await fs.writeFile(item5Path, 'item5 file') + /* Directory structure of files that were created: root/ folder-a/ @@ -93,71 +105,93 @@ describe('search', () => { item4.txt item5.txt */ - const searchResult = await findFilesToUpload(artifactName, root) - expect(searchResult.length).toEqual(5) - - const absolutePaths = searchResult.map(item => item.absoluteFilePath) - expect(absolutePaths.includes(item1Path)).toEqual(true); - expect(absolutePaths.includes(item2Path)).toEqual(true); - expect(absolutePaths.includes(item3Path)).toEqual(true); - expect(absolutePaths.includes(item4Path)).toEqual(true); - expect(absolutePaths.includes(item5Path)).toEqual(true); + const searchResult = await findFilesToUpload(artifactName, root) + expect(searchResult.length).toEqual(5) - for (const result of searchResult){ - if(result.absoluteFilePath === item1Path){ - expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-a', 'folder-b', 'folder-c', 'item1.txt')); - } - if(result.absoluteFilePath === item2Path){ - expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-d', 'item2.txt')); - } - if(result.absoluteFilePath === item3Path){ - expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-d', 'item3.txt')); - } - if(result.absoluteFilePath === item4Path){ - expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-d', 'item4.txt')); - } - if(result.absoluteFilePath === item5Path){ - expect(result.uploadFilePath).toEqual(path.join(artifactName,'item5.txt')); - } - } - }) + const absolutePaths = searchResult.map(item => item.absoluteFilePath) + expect(absolutePaths.includes(item1Path)).toEqual(true) + expect(absolutePaths.includes(item2Path)).toEqual(true) + expect(absolutePaths.includes(item3Path)).toEqual(true) + expect(absolutePaths.includes(item4Path)).toEqual(true) + expect(absolutePaths.includes(item5Path)).toEqual(true) + + for (const result of searchResult) { + if (result.absoluteFilePath === item1Path) { + expect(result.uploadFilePath).toEqual( + path.join( + artifactName, + 'folder-a', + 'folder-b', + 'folder-c', + 'item1.txt' + ) + ) + } + if (result.absoluteFilePath === item2Path) { + expect(result.uploadFilePath).toEqual( + path.join(artifactName, 'folder-d', 'item2.txt') + ) + } + if (result.absoluteFilePath === item3Path) { + expect(result.uploadFilePath).toEqual( + path.join(artifactName, 'folder-d', 'item3.txt') + ) + } + if (result.absoluteFilePath === item4Path) { + expect(result.uploadFilePath).toEqual( + path.join(artifactName, 'folder-d', 'item4.txt') + ) + } + if (result.absoluteFilePath === item5Path) { + expect(result.uploadFilePath).toEqual( + path.join(artifactName, 'item5.txt') + ) + } + } + }) - /** - * Creates a directory with multiple files and subdirectories, includes some empty directories - * All items are expected to be found - */ - it('Directory search - with empty directories', async () => { - const artifactName = "my-artifact" - const root = path.join(getTestTemp(), 'single-file-artifact') - await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-e'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-d'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-f'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-g'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-h', 'folder-i'), { - recursive: true - }) - const item1Path = path.join(root, 'folder-a', 'folder-b', 'folder-c', 'item1.txt') - const item2Path = path.join(root, 'folder-d', 'item2.txt') - const item3Path = path.join(root, 'folder-d', 'item3.txt') - const item4Path = path.join(root, 'folder-d', 'item4.txt') - const item5Path = path.join(root, 'item5.txt') - await fs.writeFile(item1Path, 'item1 file') - await fs.writeFile(item2Path, 'item2 file') - await fs.writeFile(item3Path, 'item3 file') - await fs.writeFile(item4Path, 'item4 file') - await fs.writeFile(item5Path, 'item5 file') - /* + /** + * Creates a directory with multiple files and subdirectories, includes some empty directories + * All items are expected to be found + */ + it('Directory search - with empty directories', async () => { + const artifactName = 'my-artifact' + const root = path.join(getTestTemp(), 'single-file-artifact') + await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-e'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-d'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-f'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-g'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-h', 'folder-i'), { + recursive: true + }) + const item1Path = path.join( + root, + 'folder-a', + 'folder-b', + 'folder-c', + 'item1.txt' + ) + const item2Path = path.join(root, 'folder-d', 'item2.txt') + const item3Path = path.join(root, 'folder-d', 'item3.txt') + const item4Path = path.join(root, 'folder-d', 'item4.txt') + const item5Path = path.join(root, 'item5.txt') + await fs.writeFile(item1Path, 'item1 file') + await fs.writeFile(item2Path, 'item2 file') + await fs.writeFile(item3Path, 'item3 file') + await fs.writeFile(item4Path, 'item4 file') + await fs.writeFile(item5Path, 'item5 file') + /* Directory structure of files that were created: root/ folder-a/ @@ -175,83 +209,121 @@ describe('search', () => { folder-i/ item5.txt */ - const searchResult = await findFilesToUpload(artifactName, root) - expect(searchResult.length).toEqual(5) - - const absolutePaths = searchResult.map(item => item.absoluteFilePath) - expect(absolutePaths.includes(item1Path)).toEqual(true); - expect(absolutePaths.includes(item2Path)).toEqual(true); - expect(absolutePaths.includes(item3Path)).toEqual(true); - expect(absolutePaths.includes(item4Path)).toEqual(true); - expect(absolutePaths.includes(item5Path)).toEqual(true); + const searchResult = await findFilesToUpload(artifactName, root) + expect(searchResult.length).toEqual(5) - for (const result of searchResult){ - if(result.absoluteFilePath === item1Path){ - expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-a', 'folder-b', 'folder-c', 'item1.txt')); - } - if(result.absoluteFilePath === item2Path){ - expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-d', 'item2.txt')); - } - if(result.absoluteFilePath === item3Path){ - expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-d', 'item3.txt')); - } - if(result.absoluteFilePath === item4Path){ - expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-d', 'item4.txt')); - } - if(result.absoluteFilePath === item5Path){ - expect(result.uploadFilePath).toEqual(path.join(artifactName,'item5.txt')); - } - } - }) + const absolutePaths = searchResult.map(item => item.absoluteFilePath) + expect(absolutePaths.includes(item1Path)).toEqual(true) + expect(absolutePaths.includes(item2Path)).toEqual(true) + expect(absolutePaths.includes(item3Path)).toEqual(true) + expect(absolutePaths.includes(item4Path)).toEqual(true) + expect(absolutePaths.includes(item5Path)).toEqual(true) - /** - * Creates a directory with multiple files and subdirectories, includes some empty directories and misc files - * Only files corresponding to the good* pattern should be found - */ - it('Wildcard search for mulitple files', async () => { - const artifactName = "my-artifact" - const root = path.join(getTestTemp(), 'single-file-artifact') - await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-e'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-d'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-f'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-g'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-h', 'folder-i'), { - recursive: true - }) - const goodItem1Path = path.join(root, 'folder-a', 'folder-b', 'folder-c', 'good-item1.txt') - const goodItem2Path = path.join(root, 'folder-d', 'good-item2.txt') - const goodItem3Path = path.join(root, 'folder-d', 'good-item3.txt') - const goodItem4Path = path.join(root, 'folder-d', 'good-item4.txt') - const goodItem5Path = path.join(root, 'good-item5.txt') - await fs.writeFile(goodItem1Path, 'good item1 file') - await fs.writeFile(goodItem2Path, 'good item2 file') - await fs.writeFile(goodItem3Path, 'good item3 file') - await fs.writeFile(goodItem4Path, 'good item4 file') - await fs.writeFile(goodItem5Path, 'good item5 file') + for (const result of searchResult) { + if (result.absoluteFilePath === item1Path) { + expect(result.uploadFilePath).toEqual( + path.join( + artifactName, + 'folder-a', + 'folder-b', + 'folder-c', + 'item1.txt' + ) + ) + } + if (result.absoluteFilePath === item2Path) { + expect(result.uploadFilePath).toEqual( + path.join(artifactName, 'folder-d', 'item2.txt') + ) + } + if (result.absoluteFilePath === item3Path) { + expect(result.uploadFilePath).toEqual( + path.join(artifactName, 'folder-d', 'item3.txt') + ) + } + if (result.absoluteFilePath === item4Path) { + expect(result.uploadFilePath).toEqual( + path.join(artifactName, 'folder-d', 'item4.txt') + ) + } + if (result.absoluteFilePath === item5Path) { + expect(result.uploadFilePath).toEqual( + path.join(artifactName, 'item5.txt') + ) + } + } + }) + + /** + * Creates a directory with multiple files and subdirectories, includes some empty directories and misc files + * Only files corresponding to the good* pattern should be found + */ + it('Wildcard search for mulitple files', async () => { + const artifactName = 'my-artifact' + const root = path.join(getTestTemp(), 'single-file-artifact') + await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-e'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-d'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-f'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-g'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-h', 'folder-i'), { + recursive: true + }) + const goodItem1Path = path.join( + root, + 'folder-a', + 'folder-b', + 'folder-c', + 'good-item1.txt' + ) + const goodItem2Path = path.join(root, 'folder-d', 'good-item2.txt') + const goodItem3Path = path.join(root, 'folder-d', 'good-item3.txt') + const goodItem4Path = path.join(root, 'folder-d', 'good-item4.txt') + const goodItem5Path = path.join(root, 'good-item5.txt') + await fs.writeFile(goodItem1Path, 'good item1 file') + await fs.writeFile(goodItem2Path, 'good item2 file') + await fs.writeFile(goodItem3Path, 'good item3 file') + await fs.writeFile(goodItem4Path, 'good item4 file') + await fs.writeFile(goodItem5Path, 'good item5 file') - const badItem1Path = path.join(root, 'folder-a', 'folder-b', 'folder-c', 'bad-item1.txt') - const badItem2Path = path.join(root, 'folder-d', 'bad-item2.txt') - const badItem3Path = path.join(root, 'folder-f', 'bad-item3.txt') - const badItem4Path = path.join(root, 'folder-h', 'folder-i', 'bad-item4.txt') - const badItem5Path = path.join(root, 'folder-h', 'folder-i', 'bad-item5.txt') - await fs.writeFile(badItem1Path, 'bad item1 file') - await fs.writeFile(badItem2Path, 'bad item2 file') - await fs.writeFile(badItem3Path, 'bad item3 file') - await fs.writeFile(badItem4Path, 'bad item4 file') - await fs.writeFile(badItem5Path, 'bad item5 file') + const badItem1Path = path.join( + root, + 'folder-a', + 'folder-b', + 'folder-c', + 'bad-item1.txt' + ) + const badItem2Path = path.join(root, 'folder-d', 'bad-item2.txt') + const badItem3Path = path.join(root, 'folder-f', 'bad-item3.txt') + const badItem4Path = path.join( + root, + 'folder-h', + 'folder-i', + 'bad-item4.txt' + ) + const badItem5Path = path.join( + root, + 'folder-h', + 'folder-i', + 'bad-item5.txt' + ) + await fs.writeFile(badItem1Path, 'bad item1 file') + await fs.writeFile(badItem2Path, 'bad item2 file') + await fs.writeFile(badItem3Path, 'bad item3 file') + await fs.writeFile(badItem4Path, 'bad item4 file') + await fs.writeFile(badItem5Path, 'bad item5 file') - /* + /* Directory structure of files that were created: root/ folder-a/ @@ -274,37 +346,53 @@ describe('search', () => { bad-item5.txt good-item5.txt */ - const searchPath = path.join(root, '**/good*') - const searchResult = await findFilesToUpload(artifactName, searchPath) - expect(searchResult.length).toEqual(5) - - const absolutePaths = searchResult.map(item => item.absoluteFilePath) - expect(absolutePaths.includes(goodItem1Path)).toEqual(true); - expect(absolutePaths.includes(goodItem2Path)).toEqual(true); - expect(absolutePaths.includes(goodItem3Path)).toEqual(true); - expect(absolutePaths.includes(goodItem4Path)).toEqual(true); - expect(absolutePaths.includes(goodItem5Path)).toEqual(true); + const searchPath = path.join(root, '**/good*') + const searchResult = await findFilesToUpload(artifactName, searchPath) + expect(searchResult.length).toEqual(5) - for (const result of searchResult){ - if(result.absoluteFilePath === goodItem1Path){ - expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-a', 'folder-b', 'folder-c', 'good-item1.txt')); - } - if(result.absoluteFilePath === goodItem2Path){ - expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-d', 'good-item2.txt')); - } - if(result.absoluteFilePath === goodItem3Path){ - expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-d', 'good-item3.txt')); - } - if(result.absoluteFilePath === goodItem4Path){ - expect(result.uploadFilePath).toEqual(path.join(artifactName,'folder-d', 'good-item4.txt')); - } - if(result.absoluteFilePath === goodItem5Path){ - expect(result.uploadFilePath).toEqual(path.join(artifactName, 'good-item5.txt')); - } - } - }) + const absolutePaths = searchResult.map(item => item.absoluteFilePath) + expect(absolutePaths.includes(goodItem1Path)).toEqual(true) + expect(absolutePaths.includes(goodItem2Path)).toEqual(true) + expect(absolutePaths.includes(goodItem3Path)).toEqual(true) + expect(absolutePaths.includes(goodItem4Path)).toEqual(true) + expect(absolutePaths.includes(goodItem5Path)).toEqual(true) - function getTestTemp(): string { - return path.join(__dirname, '_temp', 'artifact') + for (const result of searchResult) { + if (result.absoluteFilePath === goodItem1Path) { + expect(result.uploadFilePath).toEqual( + path.join( + artifactName, + 'folder-a', + 'folder-b', + 'folder-c', + 'good-item1.txt' + ) + ) + } + if (result.absoluteFilePath === goodItem2Path) { + expect(result.uploadFilePath).toEqual( + path.join(artifactName, 'folder-d', 'good-item2.txt') + ) + } + if (result.absoluteFilePath === goodItem3Path) { + expect(result.uploadFilePath).toEqual( + path.join(artifactName, 'folder-d', 'good-item3.txt') + ) + } + if (result.absoluteFilePath === goodItem4Path) { + expect(result.uploadFilePath).toEqual( + path.join(artifactName, 'folder-d', 'good-item4.txt') + ) + } + if (result.absoluteFilePath === goodItem5Path) { + expect(result.uploadFilePath).toEqual( + path.join(artifactName, 'good-item5.txt') + ) + } } -}) \ No newline at end of file + }) + + function getTestTemp(): string { + return path.join(__dirname, '_temp', 'artifact') + } +}) From 4431cd8b98b2f776ea0d4c69789e4e647bae3d9e Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Wed, 15 Jan 2020 00:55:32 -0500 Subject: [PATCH 10/46] Tests for artifactName --- packages/artifact/__tests__/search.test.ts | 4 +-- packages/artifact/__tests__/util.test.ts | 35 ++++++++++++++++++++++ packages/artifact/src/utils.ts | 2 +- 3 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 packages/artifact/__tests__/util.test.ts diff --git a/packages/artifact/__tests__/search.test.ts b/packages/artifact/__tests__/search.test.ts index 84c3be9610..6005fccb76 100644 --- a/packages/artifact/__tests__/search.test.ts +++ b/packages/artifact/__tests__/search.test.ts @@ -1,7 +1,7 @@ -import {findFilesToUpload} from '../src/search' +import {promises as fs} from 'fs' import * as path from 'path' +import {findFilesToUpload} from '../src/search' import * as io from '../../io/src/io' -import {promises as fs} from 'fs' describe('search', () => { // Remove temp directory after each test diff --git a/packages/artifact/__tests__/util.test.ts b/packages/artifact/__tests__/util.test.ts new file mode 100644 index 0000000000..cf1620cf42 --- /dev/null +++ b/packages/artifact/__tests__/util.test.ts @@ -0,0 +1,35 @@ +import * as utils from '../src/utils' + +describe('utils', () => { + it('Check Artifact Name', () => { + const invalidNames = [ + 'my\\artifact', + 'my/artifact', + 'my"artifact', + 'my:artifact', + 'myartifact', + 'my|artifact', + 'my*artifact', + 'my?artifact', + 'my artifact', + '' + ] + for (const invalidName of invalidNames) { + expect(() => { + utils.checkArtifactName(invalidName) + }).toThrow() + } + + const validNames = [ + 'my-normal-artifact', + 'myNormalArtifact', + 'm¥ñðrmålÄr†ï£å¢†' + ] + for (const validName of validNames) { + expect(() => { + utils.checkArtifactName(validName) + }).not.toThrow() + } + }) +}) diff --git a/packages/artifact/src/utils.ts b/packages/artifact/src/utils.ts index 41d28df3f8..6aea614600 100644 --- a/packages/artifact/src/utils.ts +++ b/packages/artifact/src/utils.ts @@ -78,7 +78,7 @@ function getWorkFlowRunId(): string { * file systems such as NTFS. To maintain platform-agnostic behavior, all characters that are not supported by an * individual filesystem/platform will not be supported on all filesystems/platforms */ -const invalidCharacters = ['\\', '/', '"', ':', '<', '>', '|', '*', '?'] +const invalidCharacters = ['\\', '/', '"', ':', '<', '>', '|', '*', '?', ' '] /** * Scans the name of the item being uploaded to make sure there are no illegal characters From 7cf5770168903337e2f4e63ff1dc2ecd294e6139 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Mon, 27 Jan 2020 23:17:23 -0500 Subject: [PATCH 11/46] Misc improvements --- packages/artifact/__tests__/search.test.ts | 222 +++++++++++++----- packages/artifact/__tests__/upload.test.ts | 75 ++++++ packages/artifact/src/artifact.ts | 60 ++--- packages/artifact/src/contracts.ts | 5 + packages/artifact/src/download-options.ts | 7 + .../src/upload-artifact-http-client.ts | 49 ++-- packages/artifact/src/upload-info.ts | 8 +- packages/artifact/src/upload-options.ts | 18 ++ packages/artifact/src/utils.ts | 32 ++- 9 files changed, 358 insertions(+), 118 deletions(-) create mode 100644 packages/artifact/__tests__/upload.test.ts create mode 100644 packages/artifact/src/download-options.ts create mode 100644 packages/artifact/src/upload-options.ts diff --git a/packages/artifact/__tests__/search.test.ts b/packages/artifact/__tests__/search.test.ts index 6005fccb76..933718d595 100644 --- a/packages/artifact/__tests__/search.test.ts +++ b/packages/artifact/__tests__/search.test.ts @@ -28,9 +28,27 @@ describe('search', () => { 'file-under-c.txt' ) await fs.writeFile(itemPath, 'sample file under folder c') + /* + Directory structure of files that were created: + root/ + folder-a/ + folder-b/ + folder-c/ + file-under-c.txt + */ const exepectedUploadFilePath = path.join(artifactName, 'file-under-c.txt') const searchResult = await findFilesToUpload(artifactName, itemPath) + /* + searchResult[] should be equal to: + [ + { + absoluteFilePath: itemPath + uploadFilePath: my-artifact/file-under-c.txt + } + ] + */ + expect(searchResult.length).toEqual(1) expect(searchResult[0].uploadFilePath).toEqual(exepectedUploadFilePath) expect(searchResult[0].absoluteFilePath).toEqual(itemPath) @@ -55,9 +73,28 @@ describe('search', () => { 'item1.txt' ) await fs.writeFile(itemPath, 'sample file under folder c') + /* + Directory structure of files that were created: + root/ + folder-a/ + folder-b/ + folder-c/ + item1.txt + */ + const searchPath = path.join(root, '**/*m1.txt') const exepectedUploadFilePath = path.join(artifactName, 'item1.txt') const searchResult = await findFilesToUpload(artifactName, searchPath) + /* + searchResult should be equal to: + [ + { + absoluteFilePath: itemPath + uploadFilePath: my-artifact/item1.txt + } + ] + */ + expect(searchResult.length).toEqual(1) expect(searchResult[0].uploadFilePath).toEqual(exepectedUploadFilePath) expect(searchResult[0].absoluteFilePath).toEqual(itemPath) @@ -93,19 +130,45 @@ describe('search', () => { await fs.writeFile(item4Path, 'item4 file') await fs.writeFile(item5Path, 'item5 file') /* - Directory structure of files that were created: - root/ - folder-a/ - folder-b/ - folder-c/ - item1.txt - folder-d/ - item2.txt - item3.txt - item4.txt - item5.txt - */ + Directory structure of files that were created: + root/ + folder-a/ + folder-b/ + folder-c/ + item1.txt + folder-d/ + item2.txt + item3.txt + item4.txt + item5.txt + */ + const searchResult = await findFilesToUpload(artifactName, root) + /* + searchResult should be equal to: + [ + { + absoluteFilePath: item1Path + uploadFilePath: my-artifact/folder-a/folder-b/folder-c/item1.txt + }, + { + absoluteFilePath: item2Path + uploadFilePath: my-artifact/folder-d/item2.txt + }, + { + absoluteFilePath: item3Path + uploadFilePath: my-artifact/folder-d/item3.txt + }, + { + absoluteFilePath: item4Path + uploadFilePath: my-artifact/folder-d/item4.txt + }, + { + absoluteFilePath: item5Path + uploadFilePath: my-artifact/item5.txt + } + ] + */ expect(searchResult.length).toEqual(5) const absolutePaths = searchResult.map(item => item.absoluteFilePath) @@ -192,24 +255,51 @@ describe('search', () => { await fs.writeFile(item4Path, 'item4 file') await fs.writeFile(item5Path, 'item5 file') /* - Directory structure of files that were created: - root/ - folder-a/ - folder-b/ - folder-c/ - item1.txt - folder-e/ - folder-d/ - item2.txt - item3.txt - item4.txt - folder-f/ - folder-g/ - folder-h/ - folder-i/ - item5.txt - */ + Directory structure of files that were created: + root/ + folder-a/ + folder-b/ + folder-c/ + item1.txt + folder-e/ + folder-d/ + item2.txt + item3.txt + item4.txt + folder-f/ + folder-g/ + folder-h/ + folder-i/ + item5.txt + */ + const searchResult = await findFilesToUpload(artifactName, root) + /* + searchResult should be equal to: + [ + { + absoluteFilePath: item1Path + uploadFilePath: my-artifact/folder-a/folder-b/folder-c/item1.txt + }, + { + absoluteFilePath: item2Path + uploadFilePath: my-artifact/folder-d/item2.txt + }, + { + absoluteFilePath: item3Path + uploadFilePath: my-artifact/folder-d/item3.txt + }, + { + absoluteFilePath: item4Path + uploadFilePath: my-artifact/folder-d/item4.txt + }, + { + absoluteFilePath: item5Path + uploadFilePath: my-artifact/item5.txt + } + ] + */ + expect(searchResult.length).toEqual(5) const absolutePaths = searchResult.map(item => item.absoluteFilePath) @@ -322,32 +412,58 @@ describe('search', () => { await fs.writeFile(badItem3Path, 'bad item3 file') await fs.writeFile(badItem4Path, 'bad item4 file') await fs.writeFile(badItem5Path, 'bad item5 file') - /* - Directory structure of files that were created: - root/ - folder-a/ - folder-b/ - folder-c/ - good-item1.txt - bad-item1.txt - folder-e/ - folder-d/ - good-item2.txt - good-item3.txt - good-item4.txt - bad-item2.txt - folder-f/ - bad-item3.txt - folder-g/ - folder-h/ - folder-i/ - bad-item4.txt - bad-item5.txt - good-item5.txt - */ + Directory structure of files that were created: + root/ + folder-a/ + folder-b/ + folder-c/ + good-item1.txt + bad-item1.txt + folder-e/ + folder-d/ + good-item2.txt + good-item3.txt + good-item4.txt + bad-item2.txt + folder-f/ + bad-item3.txt + folder-g/ + folder-h/ + folder-i/ + bad-item4.txt + bad-item5.txt + good-item5.txt + */ + const searchPath = path.join(root, '**/good*') const searchResult = await findFilesToUpload(artifactName, searchPath) + /* + searchResult should be equal to: + [ + { + absoluteFilePath: goodItem1Path + uploadFilePath: my-artifact/folder-a/folder-b/folder-c/good-item1.txt + }, + { + absoluteFilePath: goodItem2Path + uploadFilePath: my-artifact/folder-d/good-item2.txt + }, + { + absoluteFilePath: goodItem3Path + uploadFilePath: my-artifact/folder-d/good-item3.txt + }, + { + absoluteFilePath: goodItem4Path + uploadFilePath: my-artifact/folder-d/good-item4.txt + }, + { + absoluteFilePath: goodItem5Path + uploadFilePath: my-artifact/good-item5.txt + } + ] + */ + expect(searchResult.length).toEqual(5) const absolutePaths = searchResult.map(item => item.absoluteFilePath) @@ -393,6 +509,6 @@ describe('search', () => { }) function getTestTemp(): string { - return path.join(__dirname, '_temp', 'artifact') + return path.join(__dirname, '_temp', 'artifact-search') } }) diff --git a/packages/artifact/__tests__/upload.test.ts b/packages/artifact/__tests__/upload.test.ts new file mode 100644 index 0000000000..39b6ba6cc3 --- /dev/null +++ b/packages/artifact/__tests__/upload.test.ts @@ -0,0 +1,75 @@ +import * as fs from 'fs' +import * as io from '../../io/src/io' +import * as path from 'path' +import * as uploadHttpClient from '../src/upload-artifact-http-client' + +/* +These test will fail locally if as they require some env variables to be set by the runner +*/ +describe('upload-tests', () => { + /** + * Simple test to verify an artifact container can be created with the expected response + */ + it('Create artifact in file container API test', async () => { + const name = 'my-artifact-container' + const response = await uploadHttpClient.createArtifactInFileContainer(name) + + expect(response.name).toEqual(name) + expect(response.size).toEqual(-1) + expect(response.type).toEqual('actions_storage') + + const expectedResourceUrl = `${process.env['ACTIONS_RUNTIME_URL']}_apis/resources/Containers/${response.containerId}` + expect(response.fileContainerResourceUrl).toEqual(expectedResourceUrl) + }) + + /** + * Tests creating a new artifact container, uploading a small file and then associating the + * uploaded artifact with the correct size + */ + it('Upload simple file and associate artifact', async () => { + const name = 'my-artifact-with-files' + const response = await uploadHttpClient.createArtifactInFileContainer(name) + + expect(response.name).toEqual(name) + expect(response.size).toEqual(-1) + expect(response.type).toEqual('actions_storage') + + const expectedResourceUrl = `${process.env['ACTIONS_RUNTIME_URL']}_apis/resources/Containers/${response.containerId}` + expect(response.fileContainerResourceUrl).toEqual(expectedResourceUrl) + + // clear temp directory and create a simple file that will be uploaded + await io.rmRF(getTestTemp()) + await fs.promises.mkdir(getTestTemp(), {recursive: true}) + const itemPath = path.join(getTestTemp(), 'testFile.txt') + await fs.promises.writeFile( + itemPath, + 'Simple file that we will be uploading' + ) + + /** + * findFilesToUpload() from search.ts will normally return the information for what to upload. For these tests + * however, filesToUpload will be hardcoded to just test the upload APIs + */ + const filesToUpload = [ + { + absoluteFilePath: itemPath, + uploadFilePath: path.join(name, 'testFile.txt') + } + ] + + const uploadResult = await uploadHttpClient.uploadArtifactToFileContainer( + response.fileContainerResourceUrl, + filesToUpload + ) + expect(uploadResult.failedItems.length === 0) + expect(uploadResult.size).toEqual(fs.statSync(itemPath).size) + + expect(async () => { + await uploadHttpClient.patchArtifactSize(uploadResult.size, name) + }).not.toThrow() + }) + + function getTestTemp(): string { + return path.join(__dirname, '_temp', 'artifact-upload') + } +}) diff --git a/packages/artifact/src/artifact.ts b/packages/artifact/src/artifact.ts index 997436eba5..23f27e7a65 100644 --- a/packages/artifact/src/artifact.ts +++ b/packages/artifact/src/artifact.ts @@ -6,6 +6,7 @@ import { patchArtifactSize } from './upload-artifact-http-client' import {UploadInfo} from './upload-info' +import {UploadOptions} from './upload-options' import {checkArtifactName} from './utils' /** @@ -13,11 +14,13 @@ import {checkArtifactName} from './utils' * * @param name the name of the artifact, required * @param path the directory, file, or glob pattern to denote what will be uploaded, required + * @param options extra options for customizing the upload behavior * @returns single UploadInfo object */ export async function uploadArtifact( name: string, - path: string + path: string, + options?: UploadOptions ): Promise { checkArtifactName(name) @@ -27,16 +30,21 @@ export async function uploadArtifact( // Search for the items that will be uploaded const filesToUpload: SearchResult[] = await findFilesToUpload(name, path) - let reportedSize = -1 if (filesToUpload === undefined) { - core.setFailed( - `Unable to succesfully search fo files to upload with the provided path: ${path}` + throw new Error( + `Unable to succesfully search for files to upload with the provided path: ${path}` ) } else if (filesToUpload.length === 0) { core.warning( `No files were found for the provided path: ${path}. No artifacts will be uploaded.` ) + return { + artifactName: name, + artifactItems: [], + size: 0, + failedItems: [] + } } else { /** * Step 1 of 3 @@ -55,30 +63,28 @@ export async function uploadArtifact( * Step 2 of 3 * Upload each of the files that were found concurrently */ - const uploadingArtifact: Promise = Promise.resolve( - uploadArtifactToFileContainer( - response.fileContainerResourceUrl, - filesToUpload - ) + const uploadResult = await uploadArtifactToFileContainer( + response.fileContainerResourceUrl, + filesToUpload, + options + ) + // eslint-disable-next-line no-console + console.log( + `Finished uploading artifact ${name}. Reported size is ${uploadResult.size} bytes. There were ${uploadResult.failedItems.length} items that failed to upload` ) - uploadingArtifact.then(async size => { - // eslint-disable-next-line no-console - console.log( - `All files for artifact ${name} have finished uploading. Reported upload size is ${size} bytes` - ) - /** - * Step 3 of 3 - * Update the size of the artifact to indicate we are done uploading - */ - await patchArtifactSize(size, name) - reportedSize = size - }) - } - return { - artifactName: name, - artifactItems: filesToUpload.map(item => item.absoluteFilePath), - size: reportedSize + /** + * Step 3 of 3 + * Update the size of the artifact to indicate we are done uploading + */ + await patchArtifactSize(uploadResult.size, name) + + return { + artifactName: name, + artifactItems: filesToUpload.map(item => item.absoluteFilePath), + size: uploadResult.size, + failedItems: uploadResult.failedItems + } } } @@ -88,7 +94,7 @@ Downloads a single artifact associated with a run export async function downloadArtifact( name: string, path?: string, - createArtifactFolder?:boolean + options?: DownloadOptions ): Promise { TODO diff --git a/packages/artifact/src/contracts.ts b/packages/artifact/src/contracts.ts index dc97cac735..d3ede9eb77 100644 --- a/packages/artifact/src/contracts.ts +++ b/packages/artifact/src/contracts.ts @@ -26,3 +26,8 @@ export interface PatchArtifactSizeSuccessResponse { url: string uploadUrl: string } + +export interface UploadResults { + size: number + failedItems: string[] +} diff --git a/packages/artifact/src/download-options.ts b/packages/artifact/src/download-options.ts new file mode 100644 index 0000000000..e767557a27 --- /dev/null +++ b/packages/artifact/src/download-options.ts @@ -0,0 +1,7 @@ +export interface DownloadOptions { + /** + * Specifies if a folder is created for the artifact that is downloaded (contents downloaded into this folder), + * defaults to false if not specified + * */ + createArtifactFolder?: boolean +} diff --git a/packages/artifact/src/upload-artifact-http-client.ts b/packages/artifact/src/upload-artifact-http-client.ts index 0fcea6f510..d73a1a03ab 100644 --- a/packages/artifact/src/upload-artifact-http-client.ts +++ b/packages/artifact/src/upload-artifact-http-client.ts @@ -1,27 +1,30 @@ import {debug} from '@actions/core' -import {BearerCredentialHandler} from '@actions/http-client/auth' import {HttpClientResponse, HttpClient} from '@actions/http-client/index' import {IHttpClientResponse} from '@actions/http-client/interfaces' import { CreateArtifactResponse, CreateArtifactParameters, PatchArtifactSize, - PatchArtifactSizeSuccessResponse + PatchArtifactSizeSuccessResponse, + UploadResults } from './contracts' import * as fs from 'fs' import {SearchResult} from './search' +import {UploadOptions} from './upload-options' import {URL} from 'url' import { - parseEnvNumber, + createHttpClient, getArtifactUrl, + getContentRange, + getRequestOptions, isSuccessStatusCode, isRetryableStatusCode, - getRequestOptions, - getContentRange + parseEnvNumber } from './utils' const defaultChunkUploadConcurrency = 3 const defaultFileUploadConcurrency = 2 +const userAgent = 'actions/artifact' /** * Step 1 of 3 when uploading an artifact. Creates a file container for the new artifact in the remote blob storage/file service @@ -31,19 +34,14 @@ const defaultFileUploadConcurrency = 2 export async function createArtifactInFileContainer( artifactName: string ): Promise { - const token = process.env['ACTIONS_RUNTIME_TOKEN'] || '' - const bearerCredentialHandler = new BearerCredentialHandler(token) - const requestOptions = getRequestOptions() - requestOptions['Content-Type'] = 'application/json' - - const client: HttpClient = new HttpClient('actions/artifact', [ - bearerCredentialHandler - ]) + const client = createHttpClient(userAgent) const parameters: CreateArtifactParameters = { Type: 'actions_storage', Name: artifactName } const data: string = JSON.stringify(parameters, null, 2) + const requestOptions = getRequestOptions() + requestOptions['Content-Type'] = 'application/json' const rawResponse: HttpClientResponse = await client.post( getArtifactUrl(), data, @@ -72,13 +70,10 @@ export async function createArtifactInFileContainer( */ export async function uploadArtifactToFileContainer( uploadUrl: string, - filesToUpload: SearchResult[] -): Promise { - const token = process.env['ACTIONS_RUNTIME_TOKEN'] || '' - const bearerCredentialHandler = new BearerCredentialHandler(token) - const client: HttpClient = new HttpClient('actions/artifact', [ - bearerCredentialHandler - ]) + filesToUpload: SearchResult[], + options?: UploadOptions +): Promise { + const client = createHttpClient(userAgent) const FILE_CONCURRENCY = parseEnvNumber('ARTIFACT_FILE_UPLOAD_CONCURRENCY') || @@ -111,6 +106,9 @@ export async function uploadArtifactToFileContainer( }) } + // eslint-disable-next-line no-console + console.log(options) // TODO remove, temp + const parallelUploads = [...new Array(FILE_CONCURRENCY).keys()] const fileSizes: number[] = [] let uploadedFiles = 0 @@ -131,7 +129,10 @@ export async function uploadArtifactToFileContainer( const sum = fileSizes.reduce((acc, val) => acc + val) // eslint-disable-next-line no-console console.log(`Total size of all the files uploaded ${sum}`) - return sum + return { + size: sum, + failedItems: [] + } } /** @@ -259,6 +260,7 @@ export async function patchArtifactSize( size: number, artifactName: string ): Promise { + const client = createHttpClient(userAgent) const requestOptions = getRequestOptions() requestOptions['Content-Type'] = 'application/json' @@ -268,11 +270,6 @@ export async function patchArtifactSize( const parameters: PatchArtifactSize = {Size: size} const data: string = JSON.stringify(parameters, null, 2) - const token = process.env['ACTIONS_RUNTIME_TOKEN'] || '' - const bearerCredentialHandler = new BearerCredentialHandler(token) - const client: HttpClient = new HttpClient('actions/artifact', [ - bearerCredentialHandler - ]) // eslint-disable-next-line no-console console.log(`URL is ${resourceUrl.toString()}`) diff --git a/packages/artifact/src/upload-info.ts b/packages/artifact/src/upload-info.ts index 7aa1d7d49f..739e976b28 100644 --- a/packages/artifact/src/upload-info.ts +++ b/packages/artifact/src/upload-info.ts @@ -5,7 +5,7 @@ export interface UploadInfo { artifactName: string /** - * A list of items that were uploaded as part of the artifact + * A list of all items found using the provided path that are intended to be uploaded as part of the artfiact */ artifactItems: string[] @@ -13,4 +13,10 @@ export interface UploadInfo { * Total size of the artifact in bytes that was uploaded */ size: number + + /** + * A list of items that were not uploaded as part of the artifact (includes queued items that were not uploaded if + * continueOnError is set to false). This is a subset of artifactItems. + */ + failedItems: string[] } diff --git a/packages/artifact/src/upload-options.ts b/packages/artifact/src/upload-options.ts new file mode 100644 index 0000000000..6d345952df --- /dev/null +++ b/packages/artifact/src/upload-options.ts @@ -0,0 +1,18 @@ +export interface UploadOptions { + /** + * Indicates if the artifact upload should continue if file or chunk fails to upload from any error. + * If there is a error during upload, a partial artifact will always be associated and available for + * download at the end. The size reported will be the amount of storage that the user or org will be + * charged for the partial artifact. Defaults to true if not specified + * + * If set to false, and an error is encountered, all other uploads will stop and any files or chunkes + * that were queued will not be attempted to be uploaded. The partial artifact avaiable will only + * include files and chunks up until the failure + * + * If set to true and an error is encountered, the failed file will be skipped and ignored and all + * other queued files will be attempted to be uploaded. The partial artifact at the end will have all + * files with the exception of the problematic files(s)/chunks(s) that failed to upload + * + */ + continueOnError?: boolean +} diff --git a/packages/artifact/src/utils.ts b/packages/artifact/src/utils.ts index 6aea614600..cfa0a4aa37 100644 --- a/packages/artifact/src/utils.ts +++ b/packages/artifact/src/utils.ts @@ -1,5 +1,6 @@ import {debug} from '@actions/core' -import {HttpCodes} from '@actions/http-client' +import {HttpCodes, HttpClient} from '@actions/http-client' +import {BearerCredentialHandler} from '@actions/http-client/auth' import {IHeaders} from '@actions/http-client/interfaces' const apiVersion = '6.0-preview' @@ -43,16 +44,6 @@ export function getContentRange( return `bytes ${start}-${end}/${total}` } -export function getArtifactUrl(): string { - const runtimeUrl = process.env['ACTIONS_RUNTIME_URL'] - if (!runtimeUrl) { - throw new Error('Runtime url not found, unable to create artifact.') - } - const artifactUrl = `${runtimeUrl}_apis/pipelines/workflows/${getWorkFlowRunId()}/artifacts?api-version=${apiVersion}` - debug(`Artifact Url: ${artifactUrl}`) - return artifactUrl -} - export function getRequestOptions(): IHeaders { const requestOptions: IHeaders = { Accept: createAcceptHeader('application/json') @@ -64,6 +55,25 @@ export function createAcceptHeader(type: string): string { return `${type};api-version=${apiVersion}` } +export function createHttpClient(userAgent: string): HttpClient { + const token = process.env['ACTIONS_RUNTIME_TOKEN'] + if (!token) { + throw new Error('Unable to get ACTIONS_RUNTIME_TOKEN') + } + + return new HttpClient(userAgent, [new BearerCredentialHandler(token)]) +} + +export function getArtifactUrl(): string { + const runtimeUrl = process.env['ACTIONS_RUNTIME_URL'] + if (!runtimeUrl) { + throw new Error('Runtime url not found, unable to create artifact.') + } + const artifactUrl = `${runtimeUrl}_apis/pipelines/workflows/${getWorkFlowRunId()}/artifacts?api-version=${apiVersion}` + debug(`Artifact Url: ${artifactUrl}`) + return artifactUrl +} + function getWorkFlowRunId(): string { const workFlowrunId = process.env['GITHUB_RUN_ID'] || '' if (!workFlowrunId) { From b56cdb000d15b10d82a3a4ae70767f57dceaf517 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Mon, 27 Jan 2020 23:28:28 -0500 Subject: [PATCH 12/46] Hopefully fix failing tests --- packages/artifact/src/upload-artifact-http-client.ts | 7 +++---- packages/artifact/src/utils.ts | 12 ++++++------ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/packages/artifact/src/upload-artifact-http-client.ts b/packages/artifact/src/upload-artifact-http-client.ts index d73a1a03ab..edb3b2c3ae 100644 --- a/packages/artifact/src/upload-artifact-http-client.ts +++ b/packages/artifact/src/upload-artifact-http-client.ts @@ -24,7 +24,6 @@ import { const defaultChunkUploadConcurrency = 3 const defaultFileUploadConcurrency = 2 -const userAgent = 'actions/artifact' /** * Step 1 of 3 when uploading an artifact. Creates a file container for the new artifact in the remote blob storage/file service @@ -34,7 +33,7 @@ const userAgent = 'actions/artifact' export async function createArtifactInFileContainer( artifactName: string ): Promise { - const client = createHttpClient(userAgent) + const client = createHttpClient() const parameters: CreateArtifactParameters = { Type: 'actions_storage', Name: artifactName @@ -73,7 +72,7 @@ export async function uploadArtifactToFileContainer( filesToUpload: SearchResult[], options?: UploadOptions ): Promise { - const client = createHttpClient(userAgent) + const client = createHttpClient() const FILE_CONCURRENCY = parseEnvNumber('ARTIFACT_FILE_UPLOAD_CONCURRENCY') || @@ -260,7 +259,7 @@ export async function patchArtifactSize( size: number, artifactName: string ): Promise { - const client = createHttpClient(userAgent) + const client = createHttpClient() const requestOptions = getRequestOptions() requestOptions['Content-Type'] = 'application/json' diff --git a/packages/artifact/src/utils.ts b/packages/artifact/src/utils.ts index cfa0a4aa37..a0df4eea04 100644 --- a/packages/artifact/src/utils.ts +++ b/packages/artifact/src/utils.ts @@ -55,13 +55,13 @@ export function createAcceptHeader(type: string): string { return `${type};api-version=${apiVersion}` } -export function createHttpClient(userAgent: string): HttpClient { - const token = process.env['ACTIONS_RUNTIME_TOKEN'] - if (!token) { - throw new Error('Unable to get ACTIONS_RUNTIME_TOKEN') - } +export function createHttpClient(): HttpClient { + const token = process.env["ACTIONS_RUNTIME_TOKEN"] || ""; + const bearerCredentialHandler = new BearerCredentialHandler(token); - return new HttpClient(userAgent, [new BearerCredentialHandler(token)]) + return new HttpClient('action/artifact', [ + bearerCredentialHandler + ]) } export function getArtifactUrl(): string { From 6b04ca5596895336f44144ff7b3a4ed0e2bb257f Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Mon, 27 Jan 2020 23:36:05 -0500 Subject: [PATCH 13/46] =?UTF-8?q?=F0=9F=A4=9E=20this=20works?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/artifact/src/utils.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/artifact/src/utils.ts b/packages/artifact/src/utils.ts index a0df4eea04..3378105282 100644 --- a/packages/artifact/src/utils.ts +++ b/packages/artifact/src/utils.ts @@ -56,8 +56,8 @@ export function createAcceptHeader(type: string): string { } export function createHttpClient(): HttpClient { - const token = process.env["ACTIONS_RUNTIME_TOKEN"] || ""; - const bearerCredentialHandler = new BearerCredentialHandler(token); + const token = process.env["ACTIONS_RUNTIME_TOKEN"] || "" + const bearerCredentialHandler = new BearerCredentialHandler(token) return new HttpClient('action/artifact', [ bearerCredentialHandler @@ -65,10 +65,7 @@ export function createHttpClient(): HttpClient { } export function getArtifactUrl(): string { - const runtimeUrl = process.env['ACTIONS_RUNTIME_URL'] - if (!runtimeUrl) { - throw new Error('Runtime url not found, unable to create artifact.') - } + const runtimeUrl = process.env['ACTIONS_RUNTIME_URL'] || "" const artifactUrl = `${runtimeUrl}_apis/pipelines/workflows/${getWorkFlowRunId()}/artifacts?api-version=${apiVersion}` debug(`Artifact Url: ${artifactUrl}`) return artifactUrl @@ -77,6 +74,8 @@ export function getArtifactUrl(): string { function getWorkFlowRunId(): string { const workFlowrunId = process.env['GITHUB_RUN_ID'] || '' if (!workFlowrunId) { + // eslint-disable-next-line no-console + console.log(process.env) throw new Error('Unable to get workFlowRunId') } return workFlowrunId From e12553ea6f0199c96de73434ddbe5e039bb470c4 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Mon, 27 Jan 2020 23:41:19 -0500 Subject: [PATCH 14/46] Hmmm --- packages/artifact/src/utils.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/artifact/src/utils.ts b/packages/artifact/src/utils.ts index 3378105282..e338efe7f4 100644 --- a/packages/artifact/src/utils.ts +++ b/packages/artifact/src/utils.ts @@ -68,6 +68,10 @@ export function getArtifactUrl(): string { const runtimeUrl = process.env['ACTIONS_RUNTIME_URL'] || "" const artifactUrl = `${runtimeUrl}_apis/pipelines/workflows/${getWorkFlowRunId()}/artifacts?api-version=${apiVersion}` debug(`Artifact Url: ${artifactUrl}`) + // eslint-disable-next-line no-console + console.log(artifactUrl) + // eslint-disable-next-line no-console + console.log(process.env) return artifactUrl } From 83c8848bede2710011a8ff3b32b0fdd73942d798 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Mon, 27 Jan 2020 23:46:05 -0500 Subject: [PATCH 15/46] Really weird... --- packages/artifact/src/utils.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/artifact/src/utils.ts b/packages/artifact/src/utils.ts index e338efe7f4..cfbd8ac2aa 100644 --- a/packages/artifact/src/utils.ts +++ b/packages/artifact/src/utils.ts @@ -56,6 +56,8 @@ export function createAcceptHeader(type: string): string { } export function createHttpClient(): HttpClient { + // eslint-disable-next-line no-console + console.log(process.env) const token = process.env["ACTIONS_RUNTIME_TOKEN"] || "" const bearerCredentialHandler = new BearerCredentialHandler(token) From b6f59a057476a4797cf1a6beab9854295f1eaae0 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Tue, 28 Jan 2020 13:13:32 -0500 Subject: [PATCH 16/46] Update tests for util.ts --- packages/artifact/__tests__/util.test.ts | 55 ++++++++++++++- .../src/upload-artifact-http-client.ts | 40 +++++++---- packages/artifact/src/utils.ts | 70 +++++++++++-------- 3 files changed, 121 insertions(+), 44 deletions(-) diff --git a/packages/artifact/__tests__/util.test.ts b/packages/artifact/__tests__/util.test.ts index cf1620cf42..751695a2c0 100644 --- a/packages/artifact/__tests__/util.test.ts +++ b/packages/artifact/__tests__/util.test.ts @@ -1,7 +1,8 @@ import * as utils from '../src/utils' +import {HttpCodes} from '@actions/http-client' describe('utils', () => { - it('Check Artifact Name', () => { + it('Check Artifact Name for any invalid characters', () => { const invalidNames = [ 'my\\artifact', 'my/artifact', @@ -32,4 +33,56 @@ describe('utils', () => { }).not.toThrow() } }) + + it('Test constructing artifact URL', () => { + const runtimeUrl = 'https://pipelines.actions.githubusercontent.com/abcd/' + const runId = '15' + const artifactUrl = utils.getArtifactUrl(runtimeUrl, runId) + expect(artifactUrl).toEqual( + `${runtimeUrl}_apis/pipelines/workflows/${runId}/artifacts?api-version=${utils.getApiVerion()}` + ) + }) + + it('Test constucting headers with all optional parametesr', () => { + const type = 'application/json' + const size = 24 + const range = 'bytes 0-199/200' + const options = utils.getRequestOptions(type, type, size, range) + expect(Object.keys(options).length).toEqual(4) + expect(options['Accept']).toEqual( + `${type};api-version=${utils.getApiVerion()}` + ) + expect(options['Content-Type']).toEqual(type) + expect(options['Content-Length']).toEqual(size) + expect(options['Content-Range']).toEqual(range) + }) + + it('Test constucting headers with only required parameter', () => { + const type = 'application/json' + const options = utils.getRequestOptions(type) + expect(Object.keys(options).length).toEqual(1) + expect(options['Accept']).toEqual( + `${type};api-version=${utils.getApiVerion()}` + ) + }) + + it('Test Success Status Code', () => { + expect(utils.isSuccessStatusCode(HttpCodes.OK)).toEqual(true) + expect(utils.isSuccessStatusCode(201)).toEqual(true) + expect(utils.isSuccessStatusCode(299)).toEqual(true) + expect(utils.isSuccessStatusCode(HttpCodes.NotFound)).toEqual(false) + expect(utils.isSuccessStatusCode(HttpCodes.BadGateway)).toEqual(false) + expect(utils.isSuccessStatusCode(HttpCodes.Forbidden)).toEqual(false) + }) + + it('Test Retry Status Code', () => { + expect(utils.isRetryableStatusCode(HttpCodes.BadGateway)).toEqual(true) + expect(utils.isRetryableStatusCode(HttpCodes.ServiceUnavailable)).toEqual( + true + ) + expect(utils.isRetryableStatusCode(HttpCodes.GatewayTimeout)).toEqual(true) + expect(utils.isRetryableStatusCode(HttpCodes.OK)).toEqual(false) + expect(utils.isRetryableStatusCode(HttpCodes.NotFound)).toEqual(false) + expect(utils.isRetryableStatusCode(HttpCodes.Forbidden)).toEqual(false) + }) }) diff --git a/packages/artifact/src/upload-artifact-http-client.ts b/packages/artifact/src/upload-artifact-http-client.ts index edb3b2c3ae..62aa9081a3 100644 --- a/packages/artifact/src/upload-artifact-http-client.ts +++ b/packages/artifact/src/upload-artifact-http-client.ts @@ -16,9 +16,12 @@ import { createHttpClient, getArtifactUrl, getContentRange, + getRuntimeToken, + getRuntimeUrl, getRequestOptions, - isSuccessStatusCode, + getWorkFlowRunId, isRetryableStatusCode, + isSuccessStatusCode, parseEnvNumber } from './utils' @@ -33,16 +36,18 @@ const defaultFileUploadConcurrency = 2 export async function createArtifactInFileContainer( artifactName: string ): Promise { - const client = createHttpClient() + const client = createHttpClient(getRuntimeToken()) const parameters: CreateArtifactParameters = { Type: 'actions_storage', Name: artifactName } const data: string = JSON.stringify(parameters, null, 2) - const requestOptions = getRequestOptions() - requestOptions['Content-Type'] = 'application/json' + const requestOptions = getRequestOptions( + 'application/json', + 'application/json' + ) const rawResponse: HttpClientResponse = await client.post( - getArtifactUrl(), + getArtifactUrl(getRuntimeUrl(), getWorkFlowRunId()), data, requestOptions ) @@ -72,7 +77,7 @@ export async function uploadArtifactToFileContainer( filesToUpload: SearchResult[], options?: UploadOptions ): Promise { - const client = createHttpClient() + const client = createHttpClient(getRuntimeToken()) const FILE_CONCURRENCY = parseEnvNumber('ARTIFACT_FILE_UPLOAD_CONCURRENCY') || @@ -205,10 +210,12 @@ async function uploadChunk( )}` ) - const requestOptions = getRequestOptions() - requestOptions['Content-Type'] = 'application/octet-stream' - requestOptions['Content-Length'] = totalSize - requestOptions['Content-Range'] = getContentRange(start, end, totalSize) + const requestOptions = getRequestOptions( + 'application/json', + 'application/octet-stream', + totalSize, + getContentRange(start, end, totalSize) + ) const uploadChunkRequest = async (): Promise => { return await restClient.sendStream('PUT', resourceUrl, data, requestOptions) @@ -259,11 +266,14 @@ export async function patchArtifactSize( size: number, artifactName: string ): Promise { - const client = createHttpClient() - const requestOptions = getRequestOptions() - requestOptions['Content-Type'] = 'application/json' - - const resourceUrl = new URL(getArtifactUrl()) + const client = createHttpClient(getRuntimeToken()) + const requestOptions = getRequestOptions( + 'application/json', + 'application/json' + ) + const resourceUrl = new URL( + getArtifactUrl(getRuntimeUrl(), getWorkFlowRunId()) + ) resourceUrl.searchParams.append('artifactName', artifactName) const parameters: PatchArtifactSize = {Size: size} diff --git a/packages/artifact/src/utils.ts b/packages/artifact/src/utils.ts index cfbd8ac2aa..c69ab1cf6e 100644 --- a/packages/artifact/src/utils.ts +++ b/packages/artifact/src/utils.ts @@ -3,8 +3,6 @@ import {HttpCodes, HttpClient} from '@actions/http-client' import {BearerCredentialHandler} from '@actions/http-client/auth' import {IHeaders} from '@actions/http-client/interfaces' -const apiVersion = '6.0-preview' - /** * Parses a env variable that is a number */ @@ -19,6 +17,10 @@ export function parseEnvNumber(key: string): number | undefined { /** * Various utlity functions to help with the neceesary API calls */ +export function getApiVerion(): string { + return '6.0-preview' +} + export function isSuccessStatusCode(statusCode: number): boolean { return statusCode >= 200 && statusCode < 300 } @@ -44,45 +46,57 @@ export function getContentRange( return `bytes ${start}-${end}/${total}` } -export function getRequestOptions(): IHeaders { +export function getRequestOptions( + acceptType: string, + contentType?: string, + contentLenght?: number, + contentRange?: string +): IHeaders { const requestOptions: IHeaders = { - Accept: createAcceptHeader('application/json') + Accept: `${acceptType};api-version=${getApiVerion()}` + } + if (contentType) { + requestOptions['Content-Type'] = contentType + } + if (contentLenght) { + requestOptions['Content-Length'] = contentLenght + } + if (contentRange) { + requestOptions['Content-Range'] = contentRange } return requestOptions } -export function createAcceptHeader(type: string): string { - return `${type};api-version=${apiVersion}` +export function createHttpClient(token: string): HttpClient { + return new HttpClient('action/artifact', [new BearerCredentialHandler(token)]) } -export function createHttpClient(): HttpClient { - // eslint-disable-next-line no-console - console.log(process.env) - const token = process.env["ACTIONS_RUNTIME_TOKEN"] || "" - const bearerCredentialHandler = new BearerCredentialHandler(token) +export function getArtifactUrl(runtimeUrl: string, runId: string): string { + const artifactUrl = `${runtimeUrl}_apis/pipelines/workflows/${runId}/artifacts?api-version=${getApiVerion()}` + debug(`Artifact Url: ${artifactUrl}`) + return artifactUrl +} - return new HttpClient('action/artifact', [ - bearerCredentialHandler - ]) +export function getRuntimeToken(): string { + const token = process.env['ACTIONS_RUNTIME_TOKEN'] + if (!token) { + throw new Error('Unable to get ACTIONS_RUNTIME_TOKEN env variable') + } + return token } -export function getArtifactUrl(): string { - const runtimeUrl = process.env['ACTIONS_RUNTIME_URL'] || "" - const artifactUrl = `${runtimeUrl}_apis/pipelines/workflows/${getWorkFlowRunId()}/artifacts?api-version=${apiVersion}` - debug(`Artifact Url: ${artifactUrl}`) - // eslint-disable-next-line no-console - console.log(artifactUrl) - // eslint-disable-next-line no-console - console.log(process.env) - return artifactUrl +export function getRuntimeUrl(): string { + const runtimeUrl = process.env['ACTIONS_RUNTIME_URL'] + if (!runtimeUrl) { + throw new Error('Unable to get ACTIONS_RUNTIME_URL env variable') + } + return runtimeUrl } -function getWorkFlowRunId(): string { - const workFlowrunId = process.env['GITHUB_RUN_ID'] || '' +export function getWorkFlowRunId(): string { + const workFlowrunId = process.env['GITHUB_RUN_ID'] if (!workFlowrunId) { - // eslint-disable-next-line no-console - console.log(process.env) - throw new Error('Unable to get workFlowRunId') + throw new Error('Unable to get GITHUB_RUN_ID env variable') } return workFlowrunId } From c6d23c0bae8e18e444750aebec7cb931d74c7f03 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Thu, 30 Jan 2020 11:59:58 -0500 Subject: [PATCH 17/46] Succesfull http mocking along with more tests --- packages/artifact/__tests__/search.test.ts | 538 +++++------------- packages/artifact/__tests__/upload.test.ts | 203 +++++-- packages/artifact/__tests__/util.test.ts | 5 +- .../artifact/src/__mocks__/env-variables.ts | 16 + packages/artifact/src/env-variables.ts | 23 + .../src/upload-artifact-http-client.ts | 26 +- packages/artifact/src/utils.ts | 24 - 7 files changed, 354 insertions(+), 481 deletions(-) create mode 100644 packages/artifact/src/__mocks__/env-variables.ts create mode 100644 packages/artifact/src/env-variables.ts diff --git a/packages/artifact/__tests__/search.test.ts b/packages/artifact/__tests__/search.test.ts index 933718d595..40fbe11e7b 100644 --- a/packages/artifact/__tests__/search.test.ts +++ b/packages/artifact/__tests__/search.test.ts @@ -3,223 +3,43 @@ import * as path from 'path' import {findFilesToUpload} from '../src/search' import * as io from '../../io/src/io' -describe('search', () => { - // Remove temp directory after each test - afterEach(async () => { - await io.rmRF(getTestTemp()) - }) - - /** - * Creates a single item in /single-file-artifact/folder-a/folder-b/folder-b/file-under-c.txt - * Expected to find that item with full file path provided - * Only 1 item is expected to be found so the uploadFilePath is expected to be {artifactName}/file-under-c.txt - */ - it('Single file search - full path', async () => { - const artifactName = 'my-artifact' - const root = path.join(getTestTemp(), 'single-file-artifact') - await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { - recursive: true - }) - const itemPath = path.join( - root, - 'folder-a', - 'folder-b', - 'folder-c', - 'file-under-c.txt' - ) - await fs.writeFile(itemPath, 'sample file under folder c') - /* - Directory structure of files that were created: - root/ - folder-a/ - folder-b/ - folder-c/ - file-under-c.txt - */ - - const exepectedUploadFilePath = path.join(artifactName, 'file-under-c.txt') - const searchResult = await findFilesToUpload(artifactName, itemPath) - /* - searchResult[] should be equal to: - [ - { - absoluteFilePath: itemPath - uploadFilePath: my-artifact/file-under-c.txt - } - ] - */ - - expect(searchResult.length).toEqual(1) - expect(searchResult[0].uploadFilePath).toEqual(exepectedUploadFilePath) - expect(searchResult[0].absoluteFilePath).toEqual(itemPath) - }) - - /** - * Creates a single item in /single-file-artifact/folder-a/folder-b/folder-b/file-under-c.txt - * Expected to find that one item with a provided wildcard pattern - * Only 1 item is expected to be found so the uploadFilePath is expected to be {artifactName}/file-under-c.txt - */ - it('Single file search - wildcard pattern', async () => { - const artifactName = 'my-artifact' - const root = path.join(getTestTemp(), 'single-file-artifact') - await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { - recursive: true - }) - const itemPath = path.join( - root, - 'folder-a', - 'folder-b', - 'folder-c', - 'item1.txt' - ) - await fs.writeFile(itemPath, 'sample file under folder c') - /* - Directory structure of files that were created: - root/ - folder-a/ - folder-b/ - folder-c/ - item1.txt - */ - - const searchPath = path.join(root, '**/*m1.txt') - const exepectedUploadFilePath = path.join(artifactName, 'item1.txt') - const searchResult = await findFilesToUpload(artifactName, searchPath) - /* - searchResult should be equal to: - [ - { - absoluteFilePath: itemPath - uploadFilePath: my-artifact/item1.txt - } - ] - */ - - expect(searchResult.length).toEqual(1) - expect(searchResult[0].uploadFilePath).toEqual(exepectedUploadFilePath) - expect(searchResult[0].absoluteFilePath).toEqual(itemPath) - }) - - /** - * Creates a directory with multiple files and subdirectories, no empty directories - * All items are expected to be found - */ - it('Directory search - no empty directories', async () => { - const artifactName = 'my-artifact' - const root = path.join(getTestTemp(), 'single-file-artifact') - await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-d'), { - recursive: true - }) - const item1Path = path.join( - root, - 'folder-a', - 'folder-b', - 'folder-c', - 'item1.txt' - ) - const item2Path = path.join(root, 'folder-d', 'item2.txt') - const item3Path = path.join(root, 'folder-d', 'item3.txt') - const item4Path = path.join(root, 'folder-d', 'item4.txt') - const item5Path = path.join(root, 'item5.txt') - await fs.writeFile(item1Path, 'item1 file') - await fs.writeFile(item2Path, 'item2 file') - await fs.writeFile(item3Path, 'item3 file') - await fs.writeFile(item4Path, 'item4 file') - await fs.writeFile(item5Path, 'item5 file') - /* - Directory structure of files that were created: - root/ - folder-a/ - folder-b/ - folder-c/ - item1.txt - folder-d/ - item2.txt - item3.txt - item4.txt - item5.txt - */ - - const searchResult = await findFilesToUpload(artifactName, root) - /* - searchResult should be equal to: - [ - { - absoluteFilePath: item1Path - uploadFilePath: my-artifact/folder-a/folder-b/folder-c/item1.txt - }, - { - absoluteFilePath: item2Path - uploadFilePath: my-artifact/folder-d/item2.txt - }, - { - absoluteFilePath: item3Path - uploadFilePath: my-artifact/folder-d/item3.txt - }, - { - absoluteFilePath: item4Path - uploadFilePath: my-artifact/folder-d/item4.txt - }, - { - absoluteFilePath: item5Path - uploadFilePath: my-artifact/item5.txt - } - ] - */ - expect(searchResult.length).toEqual(5) +const artifactName = 'my-artifact' +const root = path.join(__dirname, '_temp', 'artifact-search') +const goodItem1Path = path.join( + root, + 'folder-a', + 'folder-b', + 'folder-c', + 'good-item1.txt' +) +const goodItem2Path = path.join(root, 'folder-d', 'good-item2.txt') +const goodItem3Path = path.join(root, 'folder-d', 'good-item3.txt') +const goodItem4Path = path.join(root, 'folder-d', 'good-item4.txt') +const goodItem5Path = path.join(root, 'good-item5.txt') +const badItem1Path = path.join( + root, + 'folder-a', + 'folder-b', + 'folder-c', + 'bad-item1.txt' +) +const badItem2Path = path.join(root, 'folder-d', 'bad-item2.txt') +const badItem3Path = path.join(root, 'folder-f', 'bad-item3.txt') +const badItem4Path = path.join(root, 'folder-h', 'folder-i', 'bad-item4.txt') +const badItem5Path = path.join(root, 'folder-h', 'folder-i', 'bad-item5.txt') +const extraFileInFolderCPath = path.join( + root, + 'folder-a', + 'folder-b', + 'folder-c', + 'extra-file-in-folder-c.txt' +) +const amazingFileinFolderHPath = path.join(root, 'folder-h', 'amazing-item.txt') - const absolutePaths = searchResult.map(item => item.absoluteFilePath) - expect(absolutePaths.includes(item1Path)).toEqual(true) - expect(absolutePaths.includes(item2Path)).toEqual(true) - expect(absolutePaths.includes(item3Path)).toEqual(true) - expect(absolutePaths.includes(item4Path)).toEqual(true) - expect(absolutePaths.includes(item5Path)).toEqual(true) - - for (const result of searchResult) { - if (result.absoluteFilePath === item1Path) { - expect(result.uploadFilePath).toEqual( - path.join( - artifactName, - 'folder-a', - 'folder-b', - 'folder-c', - 'item1.txt' - ) - ) - } - if (result.absoluteFilePath === item2Path) { - expect(result.uploadFilePath).toEqual( - path.join(artifactName, 'folder-d', 'item2.txt') - ) - } - if (result.absoluteFilePath === item3Path) { - expect(result.uploadFilePath).toEqual( - path.join(artifactName, 'folder-d', 'item3.txt') - ) - } - if (result.absoluteFilePath === item4Path) { - expect(result.uploadFilePath).toEqual( - path.join(artifactName, 'folder-d', 'item4.txt') - ) - } - if (result.absoluteFilePath === item5Path) { - expect(result.uploadFilePath).toEqual( - path.join(artifactName, 'item5.txt') - ) - } - } - }) - - /** - * Creates a directory with multiple files and subdirectories, includes some empty directories - * All items are expected to be found - */ - it('Directory search - with empty directories', async () => { - const artifactName = 'my-artifact' - const root = path.join(getTestTemp(), 'single-file-artifact') +describe('Search', () => { + beforeAll(async () => { + // clear temp directory + await io.rmRF(root) await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { recursive: true }) @@ -238,180 +58,22 @@ describe('search', () => { await fs.mkdir(path.join(root, 'folder-h', 'folder-i'), { recursive: true }) - const item1Path = path.join( - root, - 'folder-a', - 'folder-b', - 'folder-c', - 'item1.txt' - ) - const item2Path = path.join(root, 'folder-d', 'item2.txt') - const item3Path = path.join(root, 'folder-d', 'item3.txt') - const item4Path = path.join(root, 'folder-d', 'item4.txt') - const item5Path = path.join(root, 'item5.txt') - await fs.writeFile(item1Path, 'item1 file') - await fs.writeFile(item2Path, 'item2 file') - await fs.writeFile(item3Path, 'item3 file') - await fs.writeFile(item4Path, 'item4 file') - await fs.writeFile(item5Path, 'item5 file') - /* - Directory structure of files that were created: - root/ - folder-a/ - folder-b/ - folder-c/ - item1.txt - folder-e/ - folder-d/ - item2.txt - item3.txt - item4.txt - folder-f/ - folder-g/ - folder-h/ - folder-i/ - item5.txt - */ - const searchResult = await findFilesToUpload(artifactName, root) - /* - searchResult should be equal to: - [ - { - absoluteFilePath: item1Path - uploadFilePath: my-artifact/folder-a/folder-b/folder-c/item1.txt - }, - { - absoluteFilePath: item2Path - uploadFilePath: my-artifact/folder-d/item2.txt - }, - { - absoluteFilePath: item3Path - uploadFilePath: my-artifact/folder-d/item3.txt - }, - { - absoluteFilePath: item4Path - uploadFilePath: my-artifact/folder-d/item4.txt - }, - { - absoluteFilePath: item5Path - uploadFilePath: my-artifact/item5.txt - } - ] - */ - - expect(searchResult.length).toEqual(5) - - const absolutePaths = searchResult.map(item => item.absoluteFilePath) - expect(absolutePaths.includes(item1Path)).toEqual(true) - expect(absolutePaths.includes(item2Path)).toEqual(true) - expect(absolutePaths.includes(item3Path)).toEqual(true) - expect(absolutePaths.includes(item4Path)).toEqual(true) - expect(absolutePaths.includes(item5Path)).toEqual(true) - - for (const result of searchResult) { - if (result.absoluteFilePath === item1Path) { - expect(result.uploadFilePath).toEqual( - path.join( - artifactName, - 'folder-a', - 'folder-b', - 'folder-c', - 'item1.txt' - ) - ) - } - if (result.absoluteFilePath === item2Path) { - expect(result.uploadFilePath).toEqual( - path.join(artifactName, 'folder-d', 'item2.txt') - ) - } - if (result.absoluteFilePath === item3Path) { - expect(result.uploadFilePath).toEqual( - path.join(artifactName, 'folder-d', 'item3.txt') - ) - } - if (result.absoluteFilePath === item4Path) { - expect(result.uploadFilePath).toEqual( - path.join(artifactName, 'folder-d', 'item4.txt') - ) - } - if (result.absoluteFilePath === item5Path) { - expect(result.uploadFilePath).toEqual( - path.join(artifactName, 'item5.txt') - ) - } - } - }) - - /** - * Creates a directory with multiple files and subdirectories, includes some empty directories and misc files - * Only files corresponding to the good* pattern should be found - */ - it('Wildcard search for mulitple files', async () => { - const artifactName = 'my-artifact' - const root = path.join(getTestTemp(), 'single-file-artifact') - await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-e'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-d'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-f'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-g'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-h', 'folder-i'), { - recursive: true - }) - const goodItem1Path = path.join( - root, - 'folder-a', - 'folder-b', - 'folder-c', - 'good-item1.txt' - ) - const goodItem2Path = path.join(root, 'folder-d', 'good-item2.txt') - const goodItem3Path = path.join(root, 'folder-d', 'good-item3.txt') - const goodItem4Path = path.join(root, 'folder-d', 'good-item4.txt') - const goodItem5Path = path.join(root, 'good-item5.txt') await fs.writeFile(goodItem1Path, 'good item1 file') await fs.writeFile(goodItem2Path, 'good item2 file') await fs.writeFile(goodItem3Path, 'good item3 file') await fs.writeFile(goodItem4Path, 'good item4 file') await fs.writeFile(goodItem5Path, 'good item5 file') - const badItem1Path = path.join( - root, - 'folder-a', - 'folder-b', - 'folder-c', - 'bad-item1.txt' - ) - const badItem2Path = path.join(root, 'folder-d', 'bad-item2.txt') - const badItem3Path = path.join(root, 'folder-f', 'bad-item3.txt') - const badItem4Path = path.join( - root, - 'folder-h', - 'folder-i', - 'bad-item4.txt' - ) - const badItem5Path = path.join( - root, - 'folder-h', - 'folder-i', - 'bad-item5.txt' - ) await fs.writeFile(badItem1Path, 'bad item1 file') await fs.writeFile(badItem2Path, 'bad item2 file') await fs.writeFile(badItem3Path, 'bad item3 file') await fs.writeFile(badItem4Path, 'bad item4 file') await fs.writeFile(badItem5Path, 'bad item5 file') + + await fs.writeFile(extraFileInFolderCPath, 'extra file') + + await fs.writeFile(amazingFileinFolderHPath, 'amazing file') /* Directory structure of files that were created: root/ @@ -420,6 +82,7 @@ describe('search', () => { folder-c/ good-item1.txt bad-item1.txt + extra-file-in-folder-c.txt folder-e/ folder-d/ good-item2.txt @@ -430,12 +93,76 @@ describe('search', () => { bad-item3.txt folder-g/ folder-h/ + amazing-item.txt folder-i/ bad-item4.txt bad-item5.txt good-item5.txt */ + }) + afterAll(async () => { + await io.rmRF(root) + }) + + /** + * Creates a single item in /single-file-artifact/folder-a/folder-b/folder-b/file-under-c.txt + * Expected to find that item with full file path provided + * Only 1 item is expected to be found so the uploadFilePath is expected to be {artifactName}/file-under-c.txt + */ + it('Single file search - full path', async () => { + const exepectedUploadFilePath = path.join( + artifactName, + 'extra-file-in-folder-c.txt' + ) + const searchResult = await findFilesToUpload( + artifactName, + extraFileInFolderCPath + ) + /* + searchResult[] should be equal to: + [ + { + absoluteFilePath: extraFileInFolderCPath + uploadFilePath: my-artifact/extra-file-in-folder-c.txt + } + ] + */ + + expect(searchResult.length).toEqual(1) + expect(searchResult[0].uploadFilePath).toEqual(exepectedUploadFilePath) + expect(searchResult[0].absoluteFilePath).toEqual(extraFileInFolderCPath) + }) + + /** + * Creates a single item in /single-file-artifact/folder-a/folder-b/folder-b/file-under-c.txt + * Expected to find that one item with a provided wildcard pattern + * Only 1 item is expected to be found so the uploadFilePath is expected to be {artifactName}/file-under-c.txt + */ + it('Single file search - wildcard pattern', async () => { + const searchPath = path.join(root, '**/good*m1.txt') + const exepectedUploadFilePath = path.join(artifactName, 'good-item1.txt') + const searchResult = await findFilesToUpload(artifactName, searchPath) + /* + searchResult should be equal to: + [ + { + absoluteFilePath: goodItem1Path + uploadFilePath: my-artifact/good-item1.txt + } + ] + */ + + expect(searchResult.length).toEqual(1) + expect(searchResult[0].uploadFilePath).toEqual(exepectedUploadFilePath) + expect(searchResult[0].absoluteFilePath).toEqual(goodItem1Path) + }) + + /** + * Creates a directory with multiple files and subdirectories, includes some empty directories and misc files + * Only files corresponding to the good* pattern should be found + */ + it('Wildcard search for mulitple files', async () => { const searchPath = path.join(root, '**/good*') const searchResult = await findFilesToUpload(artifactName, searchPath) /* @@ -508,7 +235,56 @@ describe('search', () => { } }) - function getTestTemp(): string { - return path.join(__dirname, '_temp', 'artifact-search') - } + /** + * Creates a directory with multiple files and subdirectories, includes some empty directories + * All items are expected to be found + */ + it('Directory search - find everything', async () => { + const searchResult = await findFilesToUpload( + artifactName, + path.join(root, 'folder-h') + ) + /* + searchResult should be equal to: + [ + { + absoluteFilePath: amazingFileinFolderHPath + uploadFilePath: my-artifact/folder-h/amazing-item.txt + }, + { + absoluteFilePath: badItem4Path + uploadFilePath: my-artifact/folder-h/folder-i/bad-item4.txt + }, + { + absoluteFilePath: badItem5Path + uploadFilePath: my-artifact/folder-h/folder-i/bad-item5.txt + } + ] + */ + + expect(searchResult.length).toEqual(3) + + const absolutePaths = searchResult.map(item => item.absoluteFilePath) + expect(absolutePaths.includes(amazingFileinFolderHPath)).toEqual(true) + expect(absolutePaths.includes(badItem4Path)).toEqual(true) + expect(absolutePaths.includes(badItem5Path)).toEqual(true) + + for (const result of searchResult) { + if (result.absoluteFilePath === amazingFileinFolderHPath) { + expect(result.uploadFilePath).toEqual( + path.join(artifactName, 'amazing-item.txt') + ) + } + if (result.absoluteFilePath === badItem4Path) { + expect(result.uploadFilePath).toEqual( + path.join(artifactName, 'folder-i', 'bad-item4.txt') + ) + } + if (result.absoluteFilePath === badItem5Path) { + expect(result.uploadFilePath).toEqual( + path.join(artifactName, 'folder-i', 'bad-item5.txt') + ) + } + } + }) }) diff --git a/packages/artifact/__tests__/upload.test.ts b/packages/artifact/__tests__/upload.test.ts index 39b6ba6cc3..fbe3c8e652 100644 --- a/packages/artifact/__tests__/upload.test.ts +++ b/packages/artifact/__tests__/upload.test.ts @@ -1,75 +1,162 @@ -import * as fs from 'fs' -import * as io from '../../io/src/io' -import * as path from 'path' +import * as http from 'http' +import * as net from 'net' import * as uploadHttpClient from '../src/upload-artifact-http-client' +import {getRuntimeUrl} from '../src/env-variables' +import {HttpClient, HttpClientResponse} from '@actions/http-client/index' +import { + CreateArtifactResponse, + PatchArtifactSizeSuccessResponse +} from '../src/contracts' -/* -These test will fail locally if as they require some env variables to be set by the runner -*/ -describe('upload-tests', () => { - /** - * Simple test to verify an artifact container can be created with the expected response - */ - it('Create artifact in file container API test', async () => { - const name = 'my-artifact-container' - const response = await uploadHttpClient.createArtifactInFileContainer(name) - - expect(response.name).toEqual(name) - expect(response.size).toEqual(-1) - expect(response.type).toEqual('actions_storage') +// mock env variables that will not always be available along with certain http methods +jest.mock('../src/env-variables') +jest.mock('@actions/http-client') - const expectedResourceUrl = `${process.env['ACTIONS_RUNTIME_URL']}_apis/resources/Containers/${response.containerId}` - expect(response.fileContainerResourceUrl).toEqual(expectedResourceUrl) +describe('Upload Tests', () => { + // setup mocking for HTTP calls + beforeAll(() => { + mockHttpPostCall() + mockHttpPatchCall() }) - /** - * Tests creating a new artifact container, uploading a small file and then associating the - * uploaded artifact with the correct size - */ - it('Upload simple file and associate artifact', async () => { - const name = 'my-artifact-with-files' - const response = await uploadHttpClient.createArtifactInFileContainer(name) - - expect(response.name).toEqual(name) + it('Create Artifact - Success', async () => { + const artifactName = 'valid-artifact-name' + const response = await uploadHttpClient.createArtifactInFileContainer( + artifactName + ) + expect(response.containerId).toEqual('13') expect(response.size).toEqual(-1) + expect(response.signedContent).toEqual('false') + expect(response.fileContainerResourceUrl).toEqual( + `${getRuntimeUrl()}_apis/resources/Containers/13` + ) expect(response.type).toEqual('actions_storage') - - const expectedResourceUrl = `${process.env['ACTIONS_RUNTIME_URL']}_apis/resources/Containers/${response.containerId}` - expect(response.fileContainerResourceUrl).toEqual(expectedResourceUrl) - - // clear temp directory and create a simple file that will be uploaded - await io.rmRF(getTestTemp()) - await fs.promises.mkdir(getTestTemp(), {recursive: true}) - const itemPath = path.join(getTestTemp(), 'testFile.txt') - await fs.promises.writeFile( - itemPath, - 'Simple file that we will be uploading' + expect(response.name).toEqual(artifactName) + expect(response.url).toEqual( + `${getRuntimeUrl()}_apis/pipelines/1/runs/1/artifacts?artifactName=${artifactName}` ) + }) - /** - * findFilesToUpload() from search.ts will normally return the information for what to upload. For these tests - * however, filesToUpload will be hardcoded to just test the upload APIs - */ - const filesToUpload = [ - { - absoluteFilePath: itemPath, - uploadFilePath: path.join(name, 'testFile.txt') - } - ] - - const uploadResult = await uploadHttpClient.uploadArtifactToFileContainer( - response.fileContainerResourceUrl, - filesToUpload + it('Create Artifact - Failure', async () => { + const artifactName = 'invalid-artifact-name' + expect( + uploadHttpClient.createArtifactInFileContainer(artifactName) + ).rejects.toEqual( + new Error( + 'Non 201 status code when creating file container for new artifact' + ) ) - expect(uploadResult.failedItems.length === 0) - expect(uploadResult.size).toEqual(fs.statSync(itemPath).size) + }) + it('Associate Artifact - Success', async () => { expect(async () => { - await uploadHttpClient.patchArtifactSize(uploadResult.size, name) + await uploadHttpClient.patchArtifactSize(130, 'my-artifact') }).not.toThrow() }) - function getTestTemp(): string { - return path.join(__dirname, '_temp', 'artifact-upload') + it('Associate Artifact - Not Found', async () => { + await expect( + uploadHttpClient.patchArtifactSize(100, 'non-existent-artifact') + ).rejects.toThrow( + 'An Artifact with the name non-existent-artifact was not found' + ) + }) + + it('Associate Artifact - Error', async () => { + await expect( + uploadHttpClient.patchArtifactSize(-2, 'my-artifact') + ).rejects.toThrow('Unable to finish uploading artifact my-artifact') + }) + + async function mockReadBodyEmpty(): Promise { + return new Promise(resolve => { + resolve() + }) + } + + /** + * Mocks http post calls that are made when first creating a container for an artifact + */ + function mockHttpPostCall(): void { + // eslint-disable-next-line @typescript-eslint/unbound-method + HttpClient.prototype.post = async ( + requestdata, + data + ): Promise => { + // parse the input data and use the provided artifact name as part of the response + const inputData = JSON.parse(data) + const mockMessage = new http.IncomingMessage(new net.Socket()) + let mockReadBody = mockReadBodyEmpty + + if (inputData.Name === 'invalid-artifact-name') { + mockMessage.statusCode = 400 + } else { + mockMessage.statusCode = 201 + const response: CreateArtifactResponse = { + containerId: '13', + size: -1, + signedContent: 'false', + fileContainerResourceUrl: `${getRuntimeUrl()}_apis/resources/Containers/13`, + type: 'actions_storage', + name: inputData.Name, + url: `${getRuntimeUrl()}_apis/pipelines/1/runs/1/artifacts?artifactName=${ + inputData.Name + }` + } + const returnData: string = JSON.stringify(response, null, 2) + mockReadBody = async function(): Promise { + return new Promise(resolve => { + resolve(returnData) + }) + } + } + return new Promise(resolve => { + resolve({ + message: mockMessage, + readBody: mockReadBody + }) + }) + } + } + + /** + * Mocks http patch calls that are made at the very end of the artifact upload process to update the size + */ + function mockHttpPatchCall(): void { + // eslint-disable-next-line @typescript-eslint/unbound-method + HttpClient.prototype.patch = jest.fn(async (requestdata, data) => { + const inputData = JSON.parse(data) + const mockMessage = new http.IncomingMessage(new net.Socket()) + const artifactName = requestdata.split('=')[2] + let mockReadBody = mockReadBodyEmpty + if (inputData.Size < 1) { + mockMessage.statusCode = 400 + } else if (artifactName === 'non-existent-artifact') { + mockMessage.statusCode = 404 + } else { + mockMessage.statusCode = 200 + const response: PatchArtifactSizeSuccessResponse = { + containerId: 13, + size: inputData.Size, + signedContent: 'false', + type: 'actions_storage', + name: artifactName, + url: `${getRuntimeUrl()}_apis/pipelines/1/runs/1/artifacts?artifactName=${artifactName}`, + uploadUrl: `${getRuntimeUrl()}_apis/resources/Containers/13` + } + const returnData: string = JSON.stringify(response, null, 2) + mockReadBody = async function(): Promise { + return new Promise(resolve => { + resolve(returnData) + }) + } + } + return new Promise(resolve => { + resolve({ + message: mockMessage, + readBody: mockReadBody + }) + }) + }) } }) diff --git a/packages/artifact/__tests__/util.test.ts b/packages/artifact/__tests__/util.test.ts index 751695a2c0..b4504431a5 100644 --- a/packages/artifact/__tests__/util.test.ts +++ b/packages/artifact/__tests__/util.test.ts @@ -1,7 +1,10 @@ import * as utils from '../src/utils' import {HttpCodes} from '@actions/http-client' -describe('utils', () => { +// use the actual implementation of for these tests @actions/http-client +jest.unmock('@actions/http-client') + +describe('Utils', () => { it('Check Artifact Name for any invalid characters', () => { const invalidNames = [ 'my\\artifact', diff --git a/packages/artifact/src/__mocks__/env-variables.ts b/packages/artifact/src/__mocks__/env-variables.ts new file mode 100644 index 0000000000..4029746298 --- /dev/null +++ b/packages/artifact/src/__mocks__/env-variables.ts @@ -0,0 +1,16 @@ +/** + * Mocks the 'ACTIONS_RUNTIME_TOKEN', 'ACTIONS_RUNTIME_URL' and 'GITHUB_RUN_ID' env variables + * that are only available from a node context on the runner. This allows for tests to run + * locally without the env variables actually being set + */ +export function getRuntimeToken(): string { + return 'totally-valid-token' +} + +export function getRuntimeUrl(): string { + return 'https://www.example.com/' +} + +export function getWorkFlowRunId(): string { + return '15' +} diff --git a/packages/artifact/src/env-variables.ts b/packages/artifact/src/env-variables.ts new file mode 100644 index 0000000000..db104064b5 --- /dev/null +++ b/packages/artifact/src/env-variables.ts @@ -0,0 +1,23 @@ +export function getRuntimeToken(): string { + const token = process.env['ACTIONS_RUNTIME_TOKEN'] + if (!token) { + throw new Error('Unable to get ACTIONS_RUNTIME_TOKEN env variable') + } + return token +} + +export function getRuntimeUrl(): string { + const runtimeUrl = process.env['ACTIONS_RUNTIME_URL'] + if (!runtimeUrl) { + throw new Error('Unable to get ACTIONS_RUNTIME_URL env variable') + } + return runtimeUrl +} + +export function getWorkFlowRunId(): string { + const workFlowRunId = process.env['GITHUB_RUN_ID'] + if (!workFlowRunId) { + throw new Error('Unable to get GITHUB_RUN_ID env variable') + } + return workFlowRunId +} diff --git a/packages/artifact/src/upload-artifact-http-client.ts b/packages/artifact/src/upload-artifact-http-client.ts index 62aa9081a3..fa0c10aea3 100644 --- a/packages/artifact/src/upload-artifact-http-client.ts +++ b/packages/artifact/src/upload-artifact-http-client.ts @@ -16,14 +16,12 @@ import { createHttpClient, getArtifactUrl, getContentRange, - getRuntimeToken, - getRuntimeUrl, getRequestOptions, - getWorkFlowRunId, isRetryableStatusCode, isSuccessStatusCode, parseEnvNumber } from './utils' +import {getRuntimeToken, getRuntimeUrl, getWorkFlowRunId} from './env-variables' const defaultChunkUploadConcurrency = 3 const defaultFileUploadConcurrency = 2 @@ -36,30 +34,26 @@ const defaultFileUploadConcurrency = 2 export async function createArtifactInFileContainer( artifactName: string ): Promise { - const client = createHttpClient(getRuntimeToken()) const parameters: CreateArtifactParameters = { Type: 'actions_storage', Name: artifactName } const data: string = JSON.stringify(parameters, null, 2) + const artifactUrl = getArtifactUrl(getRuntimeUrl(), getWorkFlowRunId()) + const client = createHttpClient(getRuntimeToken()) const requestOptions = getRequestOptions( 'application/json', 'application/json' ) - const rawResponse: HttpClientResponse = await client.post( - getArtifactUrl(getRuntimeUrl(), getWorkFlowRunId()), - data, - requestOptions - ) + const rawResponse = await client.post(artifactUrl, data, requestOptions) const body: string = await rawResponse.readBody() - const response: CreateArtifactResponse = JSON.parse(body) - // eslint-disable-next-line no-console - console.log(response) - if (rawResponse.message.statusCode === 201 && response) { - return response + if (rawResponse.message.statusCode === 201 && body) { + return JSON.parse(body) } else { + // eslint-disable-next-line no-console + console.log(rawResponse) throw new Error( 'Non 201 status code when creating file container for new artifact' ) @@ -278,9 +272,7 @@ export async function patchArtifactSize( const parameters: PatchArtifactSize = {Size: size} const data: string = JSON.stringify(parameters, null, 2) - - // eslint-disable-next-line no-console - console.log(`URL is ${resourceUrl.toString()}`) + debug(`URL is ${resourceUrl.toString()}`) const rawResponse: HttpClientResponse = await client.patch( resourceUrl.toString(), diff --git a/packages/artifact/src/utils.ts b/packages/artifact/src/utils.ts index c69ab1cf6e..9513f6f7bf 100644 --- a/packages/artifact/src/utils.ts +++ b/packages/artifact/src/utils.ts @@ -77,30 +77,6 @@ export function getArtifactUrl(runtimeUrl: string, runId: string): string { return artifactUrl } -export function getRuntimeToken(): string { - const token = process.env['ACTIONS_RUNTIME_TOKEN'] - if (!token) { - throw new Error('Unable to get ACTIONS_RUNTIME_TOKEN env variable') - } - return token -} - -export function getRuntimeUrl(): string { - const runtimeUrl = process.env['ACTIONS_RUNTIME_URL'] - if (!runtimeUrl) { - throw new Error('Unable to get ACTIONS_RUNTIME_URL env variable') - } - return runtimeUrl -} - -export function getWorkFlowRunId(): string { - const workFlowrunId = process.env['GITHUB_RUN_ID'] - if (!workFlowrunId) { - throw new Error('Unable to get GITHUB_RUN_ID env variable') - } - return workFlowrunId -} - /** * Invalid characters that cannot be in the artifact name or an uploaded file. Will be rejected * from the server if attempted to be sent over. These characters are not allowed due to limitations with certain From da72c73b3f4335753a2802133304f10cfee70b23 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Thu, 30 Jan 2020 13:48:16 -0500 Subject: [PATCH 18/46] Spelling fixes and misc updates --- packages/artifact/__tests__/upload.test.ts | 4 ++-- packages/artifact/src/artifact.ts | 2 +- .../src/upload-artifact-http-client.ts | 21 +++++++++---------- packages/artifact/src/upload-info.ts | 2 +- packages/artifact/src/upload-options.ts | 4 ++-- packages/artifact/src/utils.ts | 18 ++++++++-------- 6 files changed, 25 insertions(+), 26 deletions(-) diff --git a/packages/artifact/__tests__/upload.test.ts b/packages/artifact/__tests__/upload.test.ts index fbe3c8e652..d0218a7fa1 100644 --- a/packages/artifact/__tests__/upload.test.ts +++ b/packages/artifact/__tests__/upload.test.ts @@ -40,10 +40,10 @@ describe('Upload Tests', () => { it('Create Artifact - Failure', async () => { const artifactName = 'invalid-artifact-name' expect( - uploadHttpClient.createArtifactInFileContainer(artifactName) + await uploadHttpClient.createArtifactInFileContainer(artifactName) ).rejects.toEqual( new Error( - 'Non 201 status code when creating file container for new artifact' + 'Unable to create a container for the artifact invalid-artifact-name' ) ) }) diff --git a/packages/artifact/src/artifact.ts b/packages/artifact/src/artifact.ts index 23f27e7a65..9676420df5 100644 --- a/packages/artifact/src/artifact.ts +++ b/packages/artifact/src/artifact.ts @@ -33,7 +33,7 @@ export async function uploadArtifact( if (filesToUpload === undefined) { throw new Error( - `Unable to succesfully search for files to upload with the provided path: ${path}` + `Unable to successfully search for files to upload with the provided path: ${path}` ) } else if (filesToUpload.length === 0) { core.warning( diff --git a/packages/artifact/src/upload-artifact-http-client.ts b/packages/artifact/src/upload-artifact-http-client.ts index fa0c10aea3..4c5844f672 100644 --- a/packages/artifact/src/upload-artifact-http-client.ts +++ b/packages/artifact/src/upload-artifact-http-client.ts @@ -27,9 +27,9 @@ const defaultChunkUploadConcurrency = 3 const defaultFileUploadConcurrency = 2 /** - * Step 1 of 3 when uploading an artifact. Creates a file container for the new artifact in the remote blob storage/file service + * Creates a file container for the new artifact in the remote blob storage/file service * @param {string} artifactName Name of the artifact being created - * @returns The response from the Artifact Service if the file container was succesfully created + * @returns The response from the Artifact Service if the file container was successfully created */ export async function createArtifactInFileContainer( artifactName: string @@ -49,19 +49,19 @@ export async function createArtifactInFileContainer( const rawResponse = await client.post(artifactUrl, data, requestOptions) const body: string = await rawResponse.readBody() - if (rawResponse.message.statusCode === 201 && body) { + if (rawResponse.message.statusCode && isSuccessStatusCode(rawResponse.message.statusCode) && body) { return JSON.parse(body) } else { // eslint-disable-next-line no-console console.log(rawResponse) throw new Error( - 'Non 201 status code when creating file container for new artifact' + `Unable to create a container for the artifact ${artifactName}` ) } } /** - * Step 2 of 3 when uploading an artifact. Concurrently upload all of the files in chunks + * Concurrently upload all of the files in chunks * @param {string} uploadUrl Base Url for the artifact that was created * @param {SearchResult[]} filesToUpload A list of information about the files being uploaded * @returns The size of all the files uploaded in bytes @@ -134,7 +134,7 @@ export async function uploadArtifactToFileContainer( } /** - * Asyncronously uploads a file. If the file is bigger than the max chunk size it will be uploaded via multiple calls + * Asynchronously uploads a file. If the file is bigger than the max chunk size it will be uploaded via multiple calls * @param {UploadFileParameters} parameters Information about the files that need to be uploaded * @returns The size of the file that was uploaded in bytes */ @@ -242,7 +242,7 @@ async function uploadChunk( if (!retryResponse.message.statusCode) { // eslint-disable-next-line no-console console.log(retryResponse) - throw new Error('No Status Code returne with response') + throw new Error('No Status Code returned with response') } if (isSuccessStatusCode(retryResponse.message.statusCode)) { return @@ -251,10 +251,9 @@ async function uploadChunk( } /** - * Step 3 of 3 when uploading an artifact - * Updates the size of the artifact from -1 which was initially set during step 1. Updating the size indicates that we are - * done uploading all the contents of the artifact. A server side check will be run to check that the artifact size is correct - * for billing purposes + * Updates the size of the artifact from -1 which was initially set when the container was first created for the artifact. + * Updating the size indicates that we are done uploading all the contents of the artifact. A server side check will be run + * to check that the artifact size is correct for billing purposes */ export async function patchArtifactSize( size: number, diff --git a/packages/artifact/src/upload-info.ts b/packages/artifact/src/upload-info.ts index 739e976b28..abc7ad760f 100644 --- a/packages/artifact/src/upload-info.ts +++ b/packages/artifact/src/upload-info.ts @@ -5,7 +5,7 @@ export interface UploadInfo { artifactName: string /** - * A list of all items found using the provided path that are intended to be uploaded as part of the artfiact + * A list of all items found using the provided path that are intended to be uploaded as part of the artifact */ artifactItems: string[] diff --git a/packages/artifact/src/upload-options.ts b/packages/artifact/src/upload-options.ts index 6d345952df..63d4febe8f 100644 --- a/packages/artifact/src/upload-options.ts +++ b/packages/artifact/src/upload-options.ts @@ -5,8 +5,8 @@ export interface UploadOptions { * download at the end. The size reported will be the amount of storage that the user or org will be * charged for the partial artifact. Defaults to true if not specified * - * If set to false, and an error is encountered, all other uploads will stop and any files or chunkes - * that were queued will not be attempted to be uploaded. The partial artifact avaiable will only + * If set to false, and an error is encountered, all other uploads will stop and any files or chunks + * that were queued will not be attempted to be uploaded. The partial artifact available will only * include files and chunks up until the failure * * If set to true and an error is encountered, the failed file will be skipped and ignored and all diff --git a/packages/artifact/src/utils.ts b/packages/artifact/src/utils.ts index 9513f6f7bf..ad7b4a8a87 100644 --- a/packages/artifact/src/utils.ts +++ b/packages/artifact/src/utils.ts @@ -15,9 +15,9 @@ export function parseEnvNumber(key: string): number | undefined { } /** - * Various utlity functions to help with the neceesary API calls + * Various utility functions to help with the necessary API calls */ -export function getApiVerion(): string { +export function getApiVersion(): string { return '6.0-preview' } @@ -39,7 +39,7 @@ export function getContentRange( end: number, total: number ): string { - // Format: `bytes start-end/filesize + // Format: `bytes start-end/fileSize // start and end are inclusive // For a 200 byte chunk starting at byte 0: // Content-Range: bytes 0-199/200 @@ -49,17 +49,17 @@ export function getContentRange( export function getRequestOptions( acceptType: string, contentType?: string, - contentLenght?: number, + contentLength?: number, contentRange?: string ): IHeaders { const requestOptions: IHeaders = { - Accept: `${acceptType};api-version=${getApiVerion()}` + Accept: `${acceptType};api-version=${getApiVersion()}` } if (contentType) { requestOptions['Content-Type'] = contentType } - if (contentLenght) { - requestOptions['Content-Length'] = contentLenght + if (contentLength) { + requestOptions['Content-Length'] = contentLength } if (contentRange) { requestOptions['Content-Range'] = contentRange @@ -72,7 +72,7 @@ export function createHttpClient(token: string): HttpClient { } export function getArtifactUrl(runtimeUrl: string, runId: string): string { - const artifactUrl = `${runtimeUrl}_apis/pipelines/workflows/${runId}/artifacts?api-version=${getApiVerion()}` + const artifactUrl = `${runtimeUrl}_apis/pipelines/workflows/${runId}/artifacts?api-version=${getApiVersion()}` debug(`Artifact Url: ${artifactUrl}`) return artifactUrl } @@ -81,7 +81,7 @@ export function getArtifactUrl(runtimeUrl: string, runId: string): string { * Invalid characters that cannot be in the artifact name or an uploaded file. Will be rejected * from the server if attempted to be sent over. These characters are not allowed due to limitations with certain * file systems such as NTFS. To maintain platform-agnostic behavior, all characters that are not supported by an - * individual filesystem/platform will not be supported on all filesystems/platforms + * individual filesystem/platform will not be supported on all fileSystems/platforms */ const invalidCharacters = ['\\', '/', '"', ':', '<', '>', '|', '*', '?', ' '] From 206233ad77dfb44033d4d14293f869c5b2c1d14a Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Thu, 30 Jan 2020 15:43:47 -0500 Subject: [PATCH 19/46] Logic for fast failing in the event a chunk fails --- packages/artifact/__tests__/search.test.ts | 20 +-- packages/artifact/__tests__/upload.test.ts | 8 +- packages/artifact/__tests__/util.test.ts | 15 +- .../src/upload-artifact-http-client.ts | 146 ++++++++++++------ packages/artifact/src/utils.ts | 3 +- 5 files changed, 117 insertions(+), 75 deletions(-) diff --git a/packages/artifact/__tests__/search.test.ts b/packages/artifact/__tests__/search.test.ts index 40fbe11e7b..da867cc227 100644 --- a/packages/artifact/__tests__/search.test.ts +++ b/packages/artifact/__tests__/search.test.ts @@ -34,7 +34,7 @@ const extraFileInFolderCPath = path.join( 'folder-c', 'extra-file-in-folder-c.txt' ) -const amazingFileinFolderHPath = path.join(root, 'folder-h', 'amazing-item.txt') +const amazingFileInFolderHPath = path.join(root, 'folder-h', 'amazing-item.txt') describe('Search', () => { beforeAll(async () => { @@ -73,7 +73,7 @@ describe('Search', () => { await fs.writeFile(extraFileInFolderCPath, 'extra file') - await fs.writeFile(amazingFileinFolderHPath, 'amazing file') + await fs.writeFile(amazingFileInFolderHPath, 'amazing file') /* Directory structure of files that were created: root/ @@ -111,7 +111,7 @@ describe('Search', () => { * Only 1 item is expected to be found so the uploadFilePath is expected to be {artifactName}/file-under-c.txt */ it('Single file search - full path', async () => { - const exepectedUploadFilePath = path.join( + const expectedUploadFilePath = path.join( artifactName, 'extra-file-in-folder-c.txt' ) @@ -130,7 +130,7 @@ describe('Search', () => { */ expect(searchResult.length).toEqual(1) - expect(searchResult[0].uploadFilePath).toEqual(exepectedUploadFilePath) + expect(searchResult[0].uploadFilePath).toEqual(expectedUploadFilePath) expect(searchResult[0].absoluteFilePath).toEqual(extraFileInFolderCPath) }) @@ -141,7 +141,7 @@ describe('Search', () => { */ it('Single file search - wildcard pattern', async () => { const searchPath = path.join(root, '**/good*m1.txt') - const exepectedUploadFilePath = path.join(artifactName, 'good-item1.txt') + const expectedUploadFilePath = path.join(artifactName, 'good-item1.txt') const searchResult = await findFilesToUpload(artifactName, searchPath) /* searchResult should be equal to: @@ -154,7 +154,7 @@ describe('Search', () => { */ expect(searchResult.length).toEqual(1) - expect(searchResult[0].uploadFilePath).toEqual(exepectedUploadFilePath) + expect(searchResult[0].uploadFilePath).toEqual(expectedUploadFilePath) expect(searchResult[0].absoluteFilePath).toEqual(goodItem1Path) }) @@ -162,7 +162,7 @@ describe('Search', () => { * Creates a directory with multiple files and subdirectories, includes some empty directories and misc files * Only files corresponding to the good* pattern should be found */ - it('Wildcard search for mulitple files', async () => { + it('Wildcard search for multiple files', async () => { const searchPath = path.join(root, '**/good*') const searchResult = await findFilesToUpload(artifactName, searchPath) /* @@ -248,7 +248,7 @@ describe('Search', () => { searchResult should be equal to: [ { - absoluteFilePath: amazingFileinFolderHPath + absoluteFilePath: amazingFileInFolderHPath uploadFilePath: my-artifact/folder-h/amazing-item.txt }, { @@ -265,12 +265,12 @@ describe('Search', () => { expect(searchResult.length).toEqual(3) const absolutePaths = searchResult.map(item => item.absoluteFilePath) - expect(absolutePaths.includes(amazingFileinFolderHPath)).toEqual(true) + expect(absolutePaths.includes(amazingFileInFolderHPath)).toEqual(true) expect(absolutePaths.includes(badItem4Path)).toEqual(true) expect(absolutePaths.includes(badItem5Path)).toEqual(true) for (const result of searchResult) { - if (result.absoluteFilePath === amazingFileinFolderHPath) { + if (result.absoluteFilePath === amazingFileInFolderHPath) { expect(result.uploadFilePath).toEqual( path.join(artifactName, 'amazing-item.txt') ) diff --git a/packages/artifact/__tests__/upload.test.ts b/packages/artifact/__tests__/upload.test.ts index d0218a7fa1..69bff1d6de 100644 --- a/packages/artifact/__tests__/upload.test.ts +++ b/packages/artifact/__tests__/upload.test.ts @@ -40,7 +40,7 @@ describe('Upload Tests', () => { it('Create Artifact - Failure', async () => { const artifactName = 'invalid-artifact-name' expect( - await uploadHttpClient.createArtifactInFileContainer(artifactName) + uploadHttpClient.createArtifactInFileContainer(artifactName) ).rejects.toEqual( new Error( 'Unable to create a container for the artifact invalid-artifact-name' @@ -50,12 +50,12 @@ describe('Upload Tests', () => { it('Associate Artifact - Success', async () => { expect(async () => { - await uploadHttpClient.patchArtifactSize(130, 'my-artifact') + uploadHttpClient.patchArtifactSize(130, 'my-artifact') }).not.toThrow() }) it('Associate Artifact - Not Found', async () => { - await expect( + expect( uploadHttpClient.patchArtifactSize(100, 'non-existent-artifact') ).rejects.toThrow( 'An Artifact with the name non-existent-artifact was not found' @@ -63,7 +63,7 @@ describe('Upload Tests', () => { }) it('Associate Artifact - Error', async () => { - await expect( + expect( uploadHttpClient.patchArtifactSize(-2, 'my-artifact') ).rejects.toThrow('Unable to finish uploading artifact my-artifact') }) diff --git a/packages/artifact/__tests__/util.test.ts b/packages/artifact/__tests__/util.test.ts index b4504431a5..bcde791d21 100644 --- a/packages/artifact/__tests__/util.test.ts +++ b/packages/artifact/__tests__/util.test.ts @@ -42,30 +42,29 @@ describe('Utils', () => { const runId = '15' const artifactUrl = utils.getArtifactUrl(runtimeUrl, runId) expect(artifactUrl).toEqual( - `${runtimeUrl}_apis/pipelines/workflows/${runId}/artifacts?api-version=${utils.getApiVerion()}` + `${runtimeUrl}_apis/pipelines/workflows/${runId}/artifacts?api-version=${utils.getApiVersion()}` ) }) - it('Test constucting headers with all optional parametesr', () => { + it('Test constructing headers with all optional parameters', () => { const type = 'application/json' const size = 24 const range = 'bytes 0-199/200' - const options = utils.getRequestOptions(type, type, size, range) + const options = utils.getRequestOptions(type, size, range) expect(Object.keys(options).length).toEqual(4) expect(options['Accept']).toEqual( - `${type};api-version=${utils.getApiVerion()}` + `${type};api-version=${utils.getApiVersion()}` ) expect(options['Content-Type']).toEqual(type) expect(options['Content-Length']).toEqual(size) expect(options['Content-Range']).toEqual(range) }) - it('Test constucting headers with only required parameter', () => { - const type = 'application/json' - const options = utils.getRequestOptions(type) + it('Test constructing headers with only required parameter', () => { + const options = utils.getRequestOptions() expect(Object.keys(options).length).toEqual(1) expect(options['Accept']).toEqual( - `${type};api-version=${utils.getApiVerion()}` + `application/json;api-version=${utils.getApiVersion()}` ) }) diff --git a/packages/artifact/src/upload-artifact-http-client.ts b/packages/artifact/src/upload-artifact-http-client.ts index 4c5844f672..6bc6630d7c 100644 --- a/packages/artifact/src/upload-artifact-http-client.ts +++ b/packages/artifact/src/upload-artifact-http-client.ts @@ -41,15 +41,16 @@ export async function createArtifactInFileContainer( const data: string = JSON.stringify(parameters, null, 2) const artifactUrl = getArtifactUrl(getRuntimeUrl(), getWorkFlowRunId()) const client = createHttpClient(getRuntimeToken()) - const requestOptions = getRequestOptions( - 'application/json', - 'application/json' - ) + const requestOptions = getRequestOptions('application/json') const rawResponse = await client.post(artifactUrl, data, requestOptions) const body: string = await rawResponse.readBody() - if (rawResponse.message.statusCode && isSuccessStatusCode(rawResponse.message.statusCode) && body) { + if ( + rawResponse.message.statusCode && + isSuccessStatusCode(rawResponse.message.statusCode) && + body + ) { return JSON.parse(body) } else { // eslint-disable-next-line no-console @@ -86,6 +87,7 @@ export async function uploadArtifactToFileContainer( ) const parameters: UploadFileParameters[] = [] + const continueOnError = options?.continueOnError || true // Prepare the necessary parameters to upload all the files for (const file of filesToUpload) { @@ -100,55 +102,74 @@ export async function uploadArtifactToFileContainer( resourceUrl: resourceUrl.toString(), restClient: client, concurrency: CHUNK_CONCURRENCY, - maxChunkSize: MAX_CHUNK_SIZE + maxChunkSize: MAX_CHUNK_SIZE, + continueOnError }) } - // eslint-disable-next-line no-console - console.log(options) // TODO remove, temp - const parallelUploads = [...new Array(FILE_CONCURRENCY).keys()] - const fileSizes: number[] = [] + const failedItemsToReport: string[] = [] let uploadedFiles = 0 + let fileSizes = 0 + let abortPendingFileUploads = false - // Only allow a certain amount of files to be uploaded at once, this is done to reduce errors if - // trying to upload everything at once + // Only allow a certain amount of files to be uploaded at once, this is done to reduce potential errors await Promise.all( parallelUploads.map(async () => { while (uploadedFiles < filesToUpload.length) { const currentFileParameters = parameters[uploadedFiles] uploadedFiles += 1 - fileSizes.push(await uploadFileAsync(currentFileParameters)) + if (abortPendingFileUploads) { + failedItemsToReport.push(currentFileParameters.file) + continue + } + + const uploadFileResult = await uploadFileAsync(currentFileParameters) + fileSizes += uploadFileResult.successfulUploadSize + if (uploadFileResult.isSuccess === false) { + failedItemsToReport.push(currentFileParameters.file) + if (!continueOnError) { + // Existing uploads will be able to finish however all pending uploads will fail fast + abortPendingFileUploads = true + } + } } }) ) - // Sum up all the files that were uploaded - const sum = fileSizes.reduce((acc, val) => acc + val) // eslint-disable-next-line no-console - console.log(`Total size of all the files uploaded ${sum}`) + console.log(`Total size of all the files uploaded ${fileSizes}`) return { - size: sum, - failedItems: [] + size: fileSizes, + failedItems: failedItemsToReport } } /** * Asynchronously uploads a file. If the file is bigger than the max chunk size it will be uploaded via multiple calls - * @param {UploadFileParameters} parameters Information about the files that need to be uploaded - * @returns The size of the file that was uploaded in bytes + * @param {UploadFileParameters} parameters Information about the file that needs to be uploaded + * @returns The size of the file that was uploaded in bytes along with any failed uploads */ async function uploadFileAsync( parameters: UploadFileParameters -): Promise { +): Promise { const fileSize: number = fs.statSync(parameters.file).size const parallelUploads = [...new Array(parameters.concurrency).keys()] let offset = 0 + let isUploadSuccessful = true + let failedChunkSizes = 0 + let abortFileUpload = false await Promise.all( parallelUploads.map(async () => { while (offset < fileSize) { const chunkSize = Math.min(fileSize - offset, parameters.maxChunkSize) + if (abortFileUpload) { + // if we don't want to continue on error, any pending upload chunk will be marked as failed + failedChunkSizes += chunkSize + continue + } + const start = offset const end = offset + chunkSize - 1 offset += parameters.maxChunkSize @@ -161,7 +182,7 @@ async function uploadFileAsync( } ) - await uploadChunk( + const result = await uploadChunk( parameters.restClient, parameters.resourceUrl, chunk, @@ -169,10 +190,28 @@ async function uploadFileAsync( end, fileSize ) + if (!result) { + /** + * Chunk failed to upload, report as failed but continue if desired. It is possible that part of a chunk was + * successfully uploaded so the server may report a different size for what was uploaded + **/ + + isUploadSuccessful = false + failedChunkSizes += chunkSize + if (!parameters.continueOnError) { + // Any currently uploading chunks will be able to finish, however pending chunks will not upload + // eslint-disable-next-line no-console + console.log(`Aborting upload for ${parameters.file} due to failure`) + abortFileUpload = true + } + } } }) ) - return fileSize + return { + isSuccess: isUploadSuccessful, + successfulUploadSize: fileSize - failedChunkSizes + } } /** @@ -184,6 +223,7 @@ async function uploadFileAsync( * @param {number} start Starting byte index of file that the chunk belongs to * @param {number} end Ending byte index of file that the chunk belongs to * @param {number} totalSize Total size of the file in bytes that is being uploaded + * @returns if the chunk was successfully uploaded */ async function uploadChunk( restClient: HttpClient, @@ -192,7 +232,7 @@ async function uploadChunk( start: number, end: number, totalSize: number -): Promise { +): Promise { // eslint-disable-next-line no-console console.log( `Uploading chunk of size ${end - @@ -205,7 +245,6 @@ async function uploadChunk( ) const requestOptions = getRequestOptions( - 'application/json', 'application/octet-stream', totalSize, getContentRange(start, end, totalSize) @@ -216,38 +255,40 @@ async function uploadChunk( } const response = await uploadChunkRequest() - - if (!response.message.statusCode) { - // eslint-disable-next-line no-console - console.log(response) - throw new Error('No Status Code returned with response') - } - - if (isSuccessStatusCode(response.message.statusCode)) { + if ( + response.message.statusCode && + isSuccessStatusCode(response.message.statusCode) + ) { debug( - `Chunk for ${start}:${end} was succesfully uploaded to ${resourceUrl}` + `Chunk for ${start}:${end} was successfully uploaded to ${resourceUrl}` ) - return - } - if (isRetryableStatusCode(response.message.statusCode)) { + return true + } else if ( + response.message.statusCode && + isRetryableStatusCode(response.message.statusCode) + ) { // eslint-disable-next-line no-console console.log( - `Received ${response.message.statusCode}, will retry chunk at offset ${start} after 10 seconds.` + `Received http ${response.message.statusCode} during chunk upload, will retry at offset ${start} after 10 seconds.` ) await new Promise(resolve => setTimeout(resolve, 10000)) - // eslint-disable-next-line no-console - console.log(`Retrying chunk at offset ${start}`) - const retryResponse = await uploadChunkRequest() - if (!retryResponse.message.statusCode) { + if ( + retryResponse.message.statusCode && + isSuccessStatusCode(retryResponse.message.statusCode) + ) { + return true + } else { // eslint-disable-next-line no-console - console.log(retryResponse) - throw new Error('No Status Code returned with response') - } - if (isSuccessStatusCode(retryResponse.message.statusCode)) { - return + console.log(`Unable to upload chunk even after retrying`) + return false } } + + // Upload must have failed spectacularly somehow, log full result for diagnostic purposes + // eslint-disable-next-line no-console + console.log(response) + return false } /** @@ -260,10 +301,7 @@ export async function patchArtifactSize( artifactName: string ): Promise { const client = createHttpClient(getRuntimeToken()) - const requestOptions = getRequestOptions( - 'application/json', - 'application/json' - ) + const requestOptions = getRequestOptions('application/json') const resourceUrl = new URL( getArtifactUrl(getRuntimeUrl(), getWorkFlowRunId()) ) @@ -303,4 +341,10 @@ interface UploadFileParameters { restClient: HttpClient concurrency: number maxChunkSize: number + continueOnError: boolean +} + +interface UploadFileResult { + isSuccess: boolean + successfulUploadSize: number } diff --git a/packages/artifact/src/utils.ts b/packages/artifact/src/utils.ts index ad7b4a8a87..8d06ba6bbf 100644 --- a/packages/artifact/src/utils.ts +++ b/packages/artifact/src/utils.ts @@ -47,13 +47,12 @@ export function getContentRange( } export function getRequestOptions( - acceptType: string, contentType?: string, contentLength?: number, contentRange?: string ): IHeaders { const requestOptions: IHeaders = { - Accept: `${acceptType};api-version=${getApiVersion()}` + Accept: `application/json;api-version=${getApiVersion()}` } if (contentType) { requestOptions['Content-Type'] = contentType From e6f55916d2d15c937ab05150cbf4780924485d24 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Thu, 30 Jan 2020 19:10:55 -0500 Subject: [PATCH 20/46] Tests for artifact uploads --- packages/artifact/__tests__/search.test.ts | 10 +- packages/artifact/__tests__/upload.test.ts | 238 ++++++++++++++++-- .../{env-variables.ts => config-variables.ts} | 14 ++ .../{env-variables.ts => config-variables.ts} | 12 + .../src/upload-artifact-http-client.ts | 34 +-- packages/artifact/src/upload-options.ts | 2 +- 6 files changed, 272 insertions(+), 38 deletions(-) rename packages/artifact/src/__mocks__/{env-variables.ts => config-variables.ts} (62%) rename packages/artifact/src/{env-variables.ts => config-variables.ts} (73%) diff --git a/packages/artifact/__tests__/search.test.ts b/packages/artifact/__tests__/search.test.ts index da867cc227..3b2ee06e7e 100644 --- a/packages/artifact/__tests__/search.test.ts +++ b/packages/artifact/__tests__/search.test.ts @@ -106,9 +106,8 @@ describe('Search', () => { }) /** - * Creates a single item in /single-file-artifact/folder-a/folder-b/folder-b/file-under-c.txt - * Expected to find that item with full file path provided - * Only 1 item is expected to be found so the uploadFilePath is expected to be {artifactName}/file-under-c.txt + * Expected to find one item with full file path provided + * Only 1 item is expected to be found so the uploadFilePath is expected to be {artifactName}/extra-file-in-folder-c.txt */ it('Single file search - full path', async () => { const expectedUploadFilePath = path.join( @@ -135,9 +134,8 @@ describe('Search', () => { }) /** - * Creates a single item in /single-file-artifact/folder-a/folder-b/folder-b/file-under-c.txt - * Expected to find that one item with a provided wildcard pattern - * Only 1 item is expected to be found so the uploadFilePath is expected to be {artifactName}/file-under-c.txt + * Expected to find one item with the provided wildcard pattern + * Only 1 item is expected to be found so the uploadFilePath is expected to be {artifactName}/good-item1.txt */ it('Single file search - wildcard pattern', async () => { const searchPath = path.join(root, '**/good*m1.txt') diff --git a/packages/artifact/__tests__/upload.test.ts b/packages/artifact/__tests__/upload.test.ts index 69bff1d6de..6732f8990c 100644 --- a/packages/artifact/__tests__/upload.test.ts +++ b/packages/artifact/__tests__/upload.test.ts @@ -1,24 +1,78 @@ import * as http from 'http' +import * as io from '../../io/src/io' import * as net from 'net' +import * as path from 'path' import * as uploadHttpClient from '../src/upload-artifact-http-client' -import {getRuntimeUrl} from '../src/env-variables' +import {promises as fs} from 'fs' +import {getRuntimeUrl} from '../src/config-variables' import {HttpClient, HttpClientResponse} from '@actions/http-client/index' import { CreateArtifactResponse, PatchArtifactSizeSuccessResponse } from '../src/contracts' +import {SearchResult} from '../src/search' + +const root = path.join(__dirname, '_temp', 'artifact-upload') +const file1Path = path.join(root, 'file1.txt') +const file2Path = path.join(root, 'file2.txt') +const file3Path = path.join(root, 'folder1', 'file3.txt') +const file4Path = path.join(root, 'folder1', 'file4.txt') +const file5Path = path.join(root, 'folder1', 'folder2', 'folder3', 'file5.txt') + +let file1Size = 0 +let file2Size = 0 +let file3Size = 0 +let file4Size = 0 +let file5Size = 0 // mock env variables that will not always be available along with certain http methods -jest.mock('../src/env-variables') +jest.mock('../src/config-variables') jest.mock('@actions/http-client') describe('Upload Tests', () => { - // setup mocking for HTTP calls - beforeAll(() => { - mockHttpPostCall() - mockHttpPatchCall() + // setup mocking for HTTP calls and prepare some test files for mock uploading + beforeAll(async () => { + mockHttpPost() + mockHttpSendStream() + mockHttpPatch() + + // clear temp directory and create files that will be "uploaded" + await io.rmRF(root) + await fs.mkdir(path.join(root, 'folder1', 'folder2', 'folder3'), { + recursive: true + }) + await fs.writeFile(file1Path, 'this is file 1') + await fs.writeFile(file2Path, 'this is file 2') + await fs.writeFile(file3Path, 'this is file 3') + await fs.writeFile(file4Path, 'this is file 4') + await fs.writeFile(file5Path, 'this is file 5') + /* + Directory structure for files that were created: + root/ + file1.txt + file2.txt + folder1/ + file3.txt + file4.txt + folder2/ + folder3/ + file5.txt + */ + + file1Size = (await fs.stat(file1Path)).size + file2Size = (await fs.stat(file2Path)).size + file3Size = (await fs.stat(file3Path)).size + file4Size = (await fs.stat(file4Path)).size + file5Size = (await fs.stat(file5Path)).size + }) + + afterAll(async () => { + await io.rmRF(root) }) + /** + * Artifact Creation Tests + */ it('Create Artifact - Success', async () => { const artifactName = 'valid-artifact-name' const response = await uploadHttpClient.createArtifactInFileContainer( @@ -48,6 +102,141 @@ describe('Upload Tests', () => { ) }) + /** + * Artifact Upload Tests + */ + it('Upload Artifact - Success', async () => { + /** + * Normally search.findFilesToUpload() would be used for providing information about what to upload. These tests however + * focuses solely on the upload APIs so searchResult[] will be hard-coded + */ + const artifactName = 'successful-artifact' + const searchResult: SearchResult[] = [ + { + absoluteFilePath: file1Path, + uploadFilePath: `${artifactName}/file1.txt` + }, + { + absoluteFilePath: file2Path, + uploadFilePath: `${artifactName}/file2.txt` + }, + { + absoluteFilePath: file3Path, + uploadFilePath: `${artifactName}/folder1/file3.txt` + }, + { + absoluteFilePath: file4Path, + uploadFilePath: `${artifactName}/folder1/file4.txt` + }, + { + absoluteFilePath: file5Path, + uploadFilePath: `${artifactName}/folder1/folder2/folder3/file5.txt` + } + ] + + const expectedTotalSize = + file1Size + file2Size + file3Size + file4Size + file5Size + const uploadUrl = `${getRuntimeUrl()}_apis/resources/Containers/13` + const uploadResult = await uploadHttpClient.uploadArtifactToFileContainer( + uploadUrl, + searchResult + ) + expect(uploadResult.failedItems.length).toEqual(0) + expect(uploadResult.size).toEqual(expectedTotalSize) + }) + + it('Upload Artifact - Failed Single File Upload', async () => { + const searchResult: SearchResult[] = [ + { + absoluteFilePath: file1Path, + uploadFilePath: `this-file-upload-will-fail` + } + ] + + const uploadUrl = `${getRuntimeUrl()}_apis/resources/Containers/13` + const uploadResult = await uploadHttpClient.uploadArtifactToFileContainer( + uploadUrl, + searchResult + ) + expect(uploadResult.failedItems.length).toEqual(1) + expect(uploadResult.size).toEqual(0) + }) + + it('Upload Artifact - Partial Upload Continue On Error', async () => { + const artifactName = 'partial-artifact' + const searchResult: SearchResult[] = [ + { + absoluteFilePath: file1Path, + uploadFilePath: `${artifactName}/file1.txt` + }, + { + absoluteFilePath: file2Path, + uploadFilePath: `${artifactName}/file2.txt` + }, + { + absoluteFilePath: file3Path, + uploadFilePath: `${artifactName}/folder1/file3.txt` + }, + { + absoluteFilePath: file4Path, + uploadFilePath: `this-file-upload-will-fail` + }, + { + absoluteFilePath: file5Path, + uploadFilePath: `${artifactName}/folder1/folder2/folder3/file5.txt` + } + ] + + const expectedPartialSize = file1Size + file2Size + file4Size + file5Size + const uploadUrl = `${getRuntimeUrl()}_apis/resources/Containers/13` + const uploadResult = await uploadHttpClient.uploadArtifactToFileContainer( + uploadUrl, + searchResult, + {continueOnError: true} + ) + expect(uploadResult.failedItems.length).toEqual(1) + expect(uploadResult.size).toEqual(expectedPartialSize) + }) + + it('Upload Artifact - Partial Upload Fail Fast', async () => { + const artifactName = 'partial-artifact' + const searchResult: SearchResult[] = [ + { + absoluteFilePath: file1Path, + uploadFilePath: `${artifactName}/file1.txt` + }, + { + absoluteFilePath: file2Path, + uploadFilePath: `${artifactName}/file2.txt` + }, + { + absoluteFilePath: file3Path, + uploadFilePath: `${artifactName}/folder1/file3.txt` + }, + { + absoluteFilePath: file4Path, + uploadFilePath: `this-file-upload-will-fail` + }, + { + absoluteFilePath: file5Path, + uploadFilePath: `${artifactName}/folder1/folder2/folder3/file5.txt` + } + ] + + const expectedPartialSize = file1Size + file2Size + file3Size + const uploadUrl = `${getRuntimeUrl()}_apis/resources/Containers/13` + const uploadResult = await uploadHttpClient.uploadArtifactToFileContainer( + uploadUrl, + searchResult, + {continueOnError: false} + ) + expect(uploadResult.failedItems.length).toEqual(2) + expect(uploadResult.size).toEqual(expectedPartialSize) + }) + + /** + * Artifact Association Tests + */ it('Associate Artifact - Success', async () => { expect(async () => { uploadHttpClient.patchArtifactSize(130, 'my-artifact') @@ -68,16 +257,16 @@ describe('Upload Tests', () => { ).rejects.toThrow('Unable to finish uploading artifact my-artifact') }) + /** + * Helpers used to setup mocking all the required HTTP calls + */ async function mockReadBodyEmpty(): Promise { return new Promise(resolve => { resolve() }) } - /** - * Mocks http post calls that are made when first creating a container for an artifact - */ - function mockHttpPostCall(): void { + function mockHttpPost(): void { // eslint-disable-next-line @typescript-eslint/unbound-method HttpClient.prototype.post = async ( requestdata, @@ -119,14 +308,35 @@ describe('Upload Tests', () => { } } - /** - * Mocks http patch calls that are made at the very end of the artifact upload process to update the size - */ - function mockHttpPatchCall(): void { + function mockHttpSendStream(): void { + // eslint-disable-next-line @typescript-eslint/unbound-method + HttpClient.prototype.sendStream = jest.fn( + async (verb, requestUrl, stream) => { + const mockMessage = new http.IncomingMessage(new net.Socket()) + mockMessage.statusCode = 200 + if (!stream.readable) { + throw new Error('Unable to read provided stream') + } + if (requestUrl.includes('fail')) { + mockMessage.statusCode = 500 + } + return new Promise(resolve => { + resolve({ + message: mockMessage, + readBody: mockReadBodyEmpty + }) + }) + } + ) + } + + function mockHttpPatch(): void { // eslint-disable-next-line @typescript-eslint/unbound-method HttpClient.prototype.patch = jest.fn(async (requestdata, data) => { const inputData = JSON.parse(data) const mockMessage = new http.IncomingMessage(new net.Socket()) + + // Get the name from the end of requestdata. Will be something like https://www.example.com/_apis/pipelines/workflows/15/artifacts?api-version=6.0-preview&artifactName=my-artifact const artifactName = requestdata.split('=')[2] let mockReadBody = mockReadBodyEmpty if (inputData.Size < 1) { diff --git a/packages/artifact/src/__mocks__/env-variables.ts b/packages/artifact/src/__mocks__/config-variables.ts similarity index 62% rename from packages/artifact/src/__mocks__/env-variables.ts rename to packages/artifact/src/__mocks__/config-variables.ts index 4029746298..5b5536173c 100644 --- a/packages/artifact/src/__mocks__/env-variables.ts +++ b/packages/artifact/src/__mocks__/config-variables.ts @@ -1,3 +1,17 @@ +/** + * Mocks default limits for easier testing + */ +export function getUploadFileConcurrency(): number { + return 1 +} + +export function getUploadChunkConcurrency(): number { + return 1 +} + +export function getUploadChunkSize(): number { + return 4 * 1024 * 1024 // 4 MB Chunks +} /** * Mocks the 'ACTIONS_RUNTIME_TOKEN', 'ACTIONS_RUNTIME_URL' and 'GITHUB_RUN_ID' env variables * that are only available from a node context on the runner. This allows for tests to run diff --git a/packages/artifact/src/env-variables.ts b/packages/artifact/src/config-variables.ts similarity index 73% rename from packages/artifact/src/env-variables.ts rename to packages/artifact/src/config-variables.ts index db104064b5..45022ed5d4 100644 --- a/packages/artifact/src/env-variables.ts +++ b/packages/artifact/src/config-variables.ts @@ -1,3 +1,15 @@ +export function getUploadFileConcurrency(): number { + return 2 +} + +export function getUploadChunkConcurrency(): number { + return 3 +} + +export function getUploadChunkSize(): number { + return 4 * 1024 * 1024 // 4 MB Chunks +} + export function getRuntimeToken(): string { const token = process.env['ACTIONS_RUNTIME_TOKEN'] if (!token) { diff --git a/packages/artifact/src/upload-artifact-http-client.ts b/packages/artifact/src/upload-artifact-http-client.ts index 6bc6630d7c..a3388f2554 100644 --- a/packages/artifact/src/upload-artifact-http-client.ts +++ b/packages/artifact/src/upload-artifact-http-client.ts @@ -18,13 +18,16 @@ import { getContentRange, getRequestOptions, isRetryableStatusCode, - isSuccessStatusCode, - parseEnvNumber + isSuccessStatusCode } from './utils' -import {getRuntimeToken, getRuntimeUrl, getWorkFlowRunId} from './env-variables' - -const defaultChunkUploadConcurrency = 3 -const defaultFileUploadConcurrency = 2 +import { + getRuntimeToken, + getRuntimeUrl, + getWorkFlowRunId, + getUploadChunkConcurrency, + getUploadChunkSize, + getUploadFileConcurrency +} from './config-variables' /** * Creates a file container for the new artifact in the remote blob storage/file service @@ -73,21 +76,18 @@ export async function uploadArtifactToFileContainer( options?: UploadOptions ): Promise { const client = createHttpClient(getRuntimeToken()) - - const FILE_CONCURRENCY = - parseEnvNumber('ARTIFACT_FILE_UPLOAD_CONCURRENCY') || - defaultFileUploadConcurrency - const CHUNK_CONCURRENCY = - parseEnvNumber('ARTIFACT_CHUNK_UPLOAD_CONCURRENCY') || - defaultChunkUploadConcurrency - const MAX_CHUNK_SIZE = - parseEnvNumber('ARTIFACT_UPLOAD_CHUNK_SIZE') || 4 * 1024 * 1024 // 4 MB Chunks + const FILE_CONCURRENCY = getUploadFileConcurrency() + const CHUNK_CONCURRENCY = getUploadChunkConcurrency() + const MAX_CHUNK_SIZE = getUploadChunkSize() debug( `File Concurrency: ${FILE_CONCURRENCY}, Chunk Concurrency: ${CHUNK_CONCURRENCY} and Chunk Size: ${MAX_CHUNK_SIZE}` ) const parameters: UploadFileParameters[] = [] - const continueOnError = options?.continueOnError || true + let continueOnError = true + if (options) { + continueOnError = options.continueOnError + } // Prepare the necessary parameters to upload all the files for (const file of filesToUpload) { @@ -190,12 +190,12 @@ async function uploadFileAsync( end, fileSize ) + if (!result) { /** * Chunk failed to upload, report as failed but continue if desired. It is possible that part of a chunk was * successfully uploaded so the server may report a different size for what was uploaded **/ - isUploadSuccessful = false failedChunkSizes += chunkSize if (!parameters.continueOnError) { diff --git a/packages/artifact/src/upload-options.ts b/packages/artifact/src/upload-options.ts index 63d4febe8f..0792010317 100644 --- a/packages/artifact/src/upload-options.ts +++ b/packages/artifact/src/upload-options.ts @@ -14,5 +14,5 @@ export interface UploadOptions { * files with the exception of the problematic files(s)/chunks(s) that failed to upload * */ - continueOnError?: boolean + continueOnError: boolean } From 7f482379e47723a37fe080bae1b0a2b7693641dd Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Thu, 30 Jan 2020 19:53:26 -0500 Subject: [PATCH 21/46] Minor fixes --- packages/artifact/__tests__/upload.test.ts | 69 +++++++++---------- .../src/upload-artifact-http-client.ts | 10 +-- 2 files changed, 37 insertions(+), 42 deletions(-) diff --git a/packages/artifact/__tests__/upload.test.ts b/packages/artifact/__tests__/upload.test.ts index 6732f8990c..292ca8f4f4 100644 --- a/packages/artifact/__tests__/upload.test.ts +++ b/packages/artifact/__tests__/upload.test.ts @@ -5,7 +5,7 @@ import * as path from 'path' import * as uploadHttpClient from '../src/upload-artifact-http-client' import {promises as fs} from 'fs' import {getRuntimeUrl} from '../src/config-variables' -import {HttpClient, HttpClientResponse} from '@actions/http-client/index' +import {HttpClient, HttpClientResponse} from '@actions/http-client' import { CreateArtifactResponse, PatchArtifactSizeSuccessResponse @@ -268,44 +268,43 @@ describe('Upload Tests', () => { function mockHttpPost(): void { // eslint-disable-next-line @typescript-eslint/unbound-method - HttpClient.prototype.post = async ( - requestdata, - data - ): Promise => { - // parse the input data and use the provided artifact name as part of the response - const inputData = JSON.parse(data) - const mockMessage = new http.IncomingMessage(new net.Socket()) - let mockReadBody = mockReadBodyEmpty + HttpClient.prototype.post = jest.fn( + async (requestdata, data): Promise => { + // parse the input data and use the provided artifact name as part of the response + const inputData = JSON.parse(data) + const mockMessage = new http.IncomingMessage(new net.Socket()) + let mockReadBody = mockReadBodyEmpty - if (inputData.Name === 'invalid-artifact-name') { - mockMessage.statusCode = 400 - } else { - mockMessage.statusCode = 201 - const response: CreateArtifactResponse = { - containerId: '13', - size: -1, - signedContent: 'false', - fileContainerResourceUrl: `${getRuntimeUrl()}_apis/resources/Containers/13`, - type: 'actions_storage', - name: inputData.Name, - url: `${getRuntimeUrl()}_apis/pipelines/1/runs/1/artifacts?artifactName=${ - inputData.Name - }` + if (inputData.Name === 'invalid-artifact-name') { + mockMessage.statusCode = 400 + } else { + mockMessage.statusCode = 201 + const response: CreateArtifactResponse = { + containerId: '13', + size: -1, + signedContent: 'false', + fileContainerResourceUrl: `${getRuntimeUrl()}_apis/resources/Containers/13`, + type: 'actions_storage', + name: inputData.Name, + url: `${getRuntimeUrl()}_apis/pipelines/1/runs/1/artifacts?artifactName=${ + inputData.Name + }` + } + const returnData: string = JSON.stringify(response, null, 2) + mockReadBody = async function(): Promise { + return new Promise(resolve => { + resolve(returnData) + }) + } } - const returnData: string = JSON.stringify(response, null, 2) - mockReadBody = async function(): Promise { - return new Promise(resolve => { - resolve(returnData) + return new Promise(resolve => { + resolve({ + message: mockMessage, + readBody: mockReadBody }) - } - } - return new Promise(resolve => { - resolve({ - message: mockMessage, - readBody: mockReadBody }) - }) - } + } + ) } function mockHttpSendStream(): void { diff --git a/packages/artifact/src/upload-artifact-http-client.ts b/packages/artifact/src/upload-artifact-http-client.ts index a3388f2554..e02056b895 100644 --- a/packages/artifact/src/upload-artifact-http-client.ts +++ b/packages/artifact/src/upload-artifact-http-client.ts @@ -138,7 +138,7 @@ export async function uploadArtifactToFileContainer( ) // eslint-disable-next-line no-console - console.log(`Total size of all the files uploaded ${fileSizes}`) + console.log(`Total size of all the files uploaded is ${fileSizes} bytes`) return { size: fileSizes, failedItems: failedItemsToReport @@ -320,12 +320,8 @@ export async function patchArtifactSize( if (rawResponse.message.statusCode === 200) { const successResponse: PatchArtifactSizeSuccessResponse = JSON.parse(body) - // eslint-disable-next-line no-console - console.log( - `Artifact ${artifactName} uploaded successfully, total size ${size}` - ) - // eslint-disable-next-line no-console - console.log(successResponse) + debug(`Artifact ${artifactName} uploaded successfully, total size ${size}`) + debug(successResponse.toString()) } else if (rawResponse.message.statusCode === 404) { throw new Error(`An Artifact with the name ${artifactName} was not found`) } else { From ddfdb57a686634f5cd47bca37fd875df29c62424 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Thu, 30 Jan 2020 20:09:48 -0500 Subject: [PATCH 22/46] PR Feedback --- packages/artifact/src/artifact.ts | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/packages/artifact/src/artifact.ts b/packages/artifact/src/artifact.ts index 9676420df5..e89531e287 100644 --- a/packages/artifact/src/artifact.ts +++ b/packages/artifact/src/artifact.ts @@ -31,11 +31,7 @@ export async function uploadArtifact( // Search for the items that will be uploaded const filesToUpload: SearchResult[] = await findFilesToUpload(name, path) - if (filesToUpload === undefined) { - throw new Error( - `Unable to successfully search for files to upload with the provided path: ${path}` - ) - } else if (filesToUpload.length === 0) { + if (filesToUpload.length === 0) { core.warning( `No files were found for the provided path: ${path}. No artifacts will be uploaded.` ) @@ -47,7 +43,6 @@ export async function uploadArtifact( } } else { /** - * Step 1 of 3 * Create an entry for the artifact in the file container */ const response = await createArtifactInFileContainer(name) @@ -60,7 +55,6 @@ export async function uploadArtifact( core.debug(`Upload Resource URL: ${response.fileContainerResourceUrl}`) /** - * Step 2 of 3 * Upload each of the files that were found concurrently */ const uploadResult = await uploadArtifactToFileContainer( @@ -68,16 +62,16 @@ export async function uploadArtifact( filesToUpload, options ) - // eslint-disable-next-line no-console - console.log( - `Finished uploading artifact ${name}. Reported size is ${uploadResult.size} bytes. There were ${uploadResult.failedItems.length} items that failed to upload` - ) /** - * Step 3 of 3 * Update the size of the artifact to indicate we are done uploading */ await patchArtifactSize(uploadResult.size, name) + + // eslint-disable-next-line no-console + console.log( + `Finished uploading artifact ${name}. Reported size is ${uploadResult.size} bytes. There were ${uploadResult.failedItems.length} items that failed to upload` + ) return { artifactName: name, From 0768705b8c9530b452cbbb2c881458d566b8ee64 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Thu, 30 Jan 2020 20:13:29 -0500 Subject: [PATCH 23/46] Format fix --- packages/artifact/src/artifact.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/artifact/src/artifact.ts b/packages/artifact/src/artifact.ts index e89531e287..75da42c833 100644 --- a/packages/artifact/src/artifact.ts +++ b/packages/artifact/src/artifact.ts @@ -67,7 +67,7 @@ export async function uploadArtifact( * Update the size of the artifact to indicate we are done uploading */ await patchArtifactSize(uploadResult.size, name) - + // eslint-disable-next-line no-console console.log( `Finished uploading artifact ${name}. Reported size is ${uploadResult.size} bytes. There were ${uploadResult.failedItems.length} items that failed to upload` From 31ff9e6173bf79e942e931ffa0fd37e9feda3d70 Mon Sep 17 00:00:00 2001 From: konradpabjan Date: Fri, 31 Jan 2020 22:30:18 -0500 Subject: [PATCH 24/46] Fixes error with unbound methods being used in tests --- packages/artifact/__tests__/upload.test.ts | 123 +++++++++++---------- 1 file changed, 67 insertions(+), 56 deletions(-) diff --git a/packages/artifact/__tests__/upload.test.ts b/packages/artifact/__tests__/upload.test.ts index 292ca8f4f4..e6c3f97446 100644 --- a/packages/artifact/__tests__/upload.test.ts +++ b/packages/artifact/__tests__/upload.test.ts @@ -25,16 +25,13 @@ let file3Size = 0 let file4Size = 0 let file5Size = 0 -// mock env variables that will not always be available along with certain http methods jest.mock('../src/config-variables') jest.mock('@actions/http-client') describe('Upload Tests', () => { - // setup mocking for HTTP calls and prepare some test files for mock uploading beforeAll(async () => { - mockHttpPost() - mockHttpSendStream() - mockHttpPatch() + // setup mocking for calls that got through the HttpClient + setupHttpClientMock() // clear temp directory and create files that will be "uploaded" await io.rmRF(root) @@ -67,6 +64,7 @@ describe('Upload Tests', () => { }) afterAll(async () => { + jest.resetAllMocks() await io.rmRF(root) }) @@ -258,22 +256,28 @@ describe('Upload Tests', () => { }) /** - * Helpers used to setup mocking all the required HTTP calls + * Helpers used to setup mocking for the HttpClient */ - async function mockReadBodyEmpty(): Promise { + async function emptyMockReadBody(): Promise { return new Promise(resolve => { resolve() }) } - function mockHttpPost(): void { - // eslint-disable-next-line @typescript-eslint/unbound-method - HttpClient.prototype.post = jest.fn( - async (requestdata, data): Promise => { + function setupHttpClientMock(): void { + /** + * Mocks Post calls that are used during Artifact Creation tests + * + * Simulates success and non-success status codes depending on the artifact name along with an appropriate + * payload that represents an expected response + */ + jest + .spyOn(HttpClient.prototype, 'post') + .mockImplementation(async (requestdata, data) => { // parse the input data and use the provided artifact name as part of the response const inputData = JSON.parse(data) const mockMessage = new http.IncomingMessage(new net.Socket()) - let mockReadBody = mockReadBodyEmpty + let mockReadBody = emptyMockReadBody if (inputData.Name === 'invalid-artifact-name') { mockMessage.statusCode = 400 @@ -303,14 +307,17 @@ describe('Upload Tests', () => { readBody: mockReadBody }) }) - } - ) - } + }) - function mockHttpSendStream(): void { - // eslint-disable-next-line @typescript-eslint/unbound-method - HttpClient.prototype.sendStream = jest.fn( - async (verb, requestUrl, stream) => { + /** + * Mocks SendStream calls that are made during Artifact Upload tests + * + * A 500 response is used to simulate a failed upload stream. The uploadUrl can be set to + * include 'fail' to specify that the upload should fail + */ + jest + .spyOn(HttpClient.prototype, 'sendStream') + .mockImplementation(async (verb, requestUrl, stream) => { const mockMessage = new http.IncomingMessage(new net.Socket()) mockMessage.statusCode = 200 if (!stream.readable) { @@ -322,50 +329,54 @@ describe('Upload Tests', () => { return new Promise(resolve => { resolve({ message: mockMessage, - readBody: mockReadBodyEmpty + readBody: emptyMockReadBody }) }) - } - ) - } + }) - function mockHttpPatch(): void { - // eslint-disable-next-line @typescript-eslint/unbound-method - HttpClient.prototype.patch = jest.fn(async (requestdata, data) => { - const inputData = JSON.parse(data) - const mockMessage = new http.IncomingMessage(new net.Socket()) + /** + * Mocks Patch calls that are made during Artifact Association tests + * + * Simulates success and non-success status codes depending on the input size along with an appropriate + * payload that represents an expected response + */ + jest + .spyOn(HttpClient.prototype, 'patch') + .mockImplementation(async (requestdata, data) => { + const inputData = JSON.parse(data) + const mockMessage = new http.IncomingMessage(new net.Socket()) - // Get the name from the end of requestdata. Will be something like https://www.example.com/_apis/pipelines/workflows/15/artifacts?api-version=6.0-preview&artifactName=my-artifact - const artifactName = requestdata.split('=')[2] - let mockReadBody = mockReadBodyEmpty - if (inputData.Size < 1) { - mockMessage.statusCode = 400 - } else if (artifactName === 'non-existent-artifact') { - mockMessage.statusCode = 404 - } else { - mockMessage.statusCode = 200 - const response: PatchArtifactSizeSuccessResponse = { - containerId: 13, - size: inputData.Size, - signedContent: 'false', - type: 'actions_storage', - name: artifactName, - url: `${getRuntimeUrl()}_apis/pipelines/1/runs/1/artifacts?artifactName=${artifactName}`, - uploadUrl: `${getRuntimeUrl()}_apis/resources/Containers/13` + // Get the name from the end of requestdata. Will be something like https://www.example.com/_apis/pipelines/workflows/15/artifacts?api-version=6.0-preview&artifactName=my-artifact + const artifactName = requestdata.split('=')[2] + let mockReadBody = emptyMockReadBody + if (inputData.Size < 1) { + mockMessage.statusCode = 400 + } else if (artifactName === 'non-existent-artifact') { + mockMessage.statusCode = 404 + } else { + mockMessage.statusCode = 200 + const response: PatchArtifactSizeSuccessResponse = { + containerId: 13, + size: inputData.Size, + signedContent: 'false', + type: 'actions_storage', + name: artifactName, + url: `${getRuntimeUrl()}_apis/pipelines/1/runs/1/artifacts?artifactName=${artifactName}`, + uploadUrl: `${getRuntimeUrl()}_apis/resources/Containers/13` + } + const returnData: string = JSON.stringify(response, null, 2) + mockReadBody = async function(): Promise { + return new Promise(resolve => { + resolve(returnData) + }) + } } - const returnData: string = JSON.stringify(response, null, 2) - mockReadBody = async function(): Promise { - return new Promise(resolve => { - resolve(returnData) + return new Promise(resolve => { + resolve({ + message: mockMessage, + readBody: mockReadBody }) - } - } - return new Promise(resolve => { - resolve({ - message: mockMessage, - readBody: mockReadBody }) }) - }) } }) From c5216d212ffec2d3de6059e6fba195a293ea2db8 Mon Sep 17 00:00:00 2001 From: konradpabjan Date: Fri, 31 Jan 2020 22:57:27 -0500 Subject: [PATCH 25/46] . --- packages/artifact/__tests__/upload.test.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/artifact/__tests__/upload.test.ts b/packages/artifact/__tests__/upload.test.ts index e6c3f97446..e3cadb9a64 100644 --- a/packages/artifact/__tests__/upload.test.ts +++ b/packages/artifact/__tests__/upload.test.ts @@ -317,15 +317,13 @@ describe('Upload Tests', () => { */ jest .spyOn(HttpClient.prototype, 'sendStream') - .mockImplementation(async (verb, requestUrl, stream) => { + .mockImplementation(async (verb, requestUrl) => { const mockMessage = new http.IncomingMessage(new net.Socket()) mockMessage.statusCode = 200 - if (!stream.readable) { - throw new Error('Unable to read provided stream') - } if (requestUrl.includes('fail')) { mockMessage.statusCode = 500 } + return new Promise(resolve => { resolve({ message: mockMessage, From a375c575386af3fba43096b84791fdb547c4f3a0 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Mon, 3 Feb 2020 01:58:17 -0500 Subject: [PATCH 26/46] Remove directory deletion after tests finish --- packages/artifact/__tests__/search.test.ts | 6 +----- packages/artifact/__tests__/upload.test.ts | 7 +------ 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/packages/artifact/__tests__/search.test.ts b/packages/artifact/__tests__/search.test.ts index 3b2ee06e7e..b6ad89662a 100644 --- a/packages/artifact/__tests__/search.test.ts +++ b/packages/artifact/__tests__/search.test.ts @@ -75,7 +75,7 @@ describe('Search', () => { await fs.writeFile(amazingFileInFolderHPath, 'amazing file') /* - Directory structure of files that were created: + Directory structure of files that get created: root/ folder-a/ folder-b/ @@ -101,10 +101,6 @@ describe('Search', () => { */ }) - afterAll(async () => { - await io.rmRF(root) - }) - /** * Expected to find one item with full file path provided * Only 1 item is expected to be found so the uploadFilePath is expected to be {artifactName}/extra-file-in-folder-c.txt diff --git a/packages/artifact/__tests__/upload.test.ts b/packages/artifact/__tests__/upload.test.ts index e3cadb9a64..531e330d01 100644 --- a/packages/artifact/__tests__/upload.test.ts +++ b/packages/artifact/__tests__/upload.test.ts @@ -44,7 +44,7 @@ describe('Upload Tests', () => { await fs.writeFile(file4Path, 'this is file 4') await fs.writeFile(file5Path, 'this is file 5') /* - Directory structure for files that were created: + Directory structure for files that get created: root/ file1.txt file2.txt @@ -63,11 +63,6 @@ describe('Upload Tests', () => { file5Size = (await fs.stat(file5Path)).size }) - afterAll(async () => { - jest.resetAllMocks() - await io.rmRF(root) - }) - /** * Artifact Creation Tests */ From c71237bcc2a9f2ef148fc8bca8b7d2139234e3de Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Mon, 3 Feb 2020 10:09:55 -0500 Subject: [PATCH 27/46] Remove project GUID from API call --- packages/artifact/src/upload-artifact-http-client.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/artifact/src/upload-artifact-http-client.ts b/packages/artifact/src/upload-artifact-http-client.ts index e02056b895..eb7f56932d 100644 --- a/packages/artifact/src/upload-artifact-http-client.ts +++ b/packages/artifact/src/upload-artifact-http-client.ts @@ -92,10 +92,6 @@ export async function uploadArtifactToFileContainer( // Prepare the necessary parameters to upload all the files for (const file of filesToUpload) { const resourceUrl = new URL(uploadUrl) - resourceUrl.searchParams.append( - 'scope', - '00000000-0000-0000-0000-000000000000' - ) resourceUrl.searchParams.append('itemPath', file.uploadFilePath) parameters.push({ file: file.absoluteFilePath, From 0cc46af350b3e842371dc3085d8394429a9cb09b Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Mon, 3 Feb 2020 10:49:23 -0500 Subject: [PATCH 28/46] Misc Improvements --- packages/artifact/src/artifact.ts | 22 +++++++++---------- .../src/upload-artifact-http-client.ts | 13 +++++++---- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/packages/artifact/src/artifact.ts b/packages/artifact/src/artifact.ts index 75da42c833..89314f73dc 100644 --- a/packages/artifact/src/artifact.ts +++ b/packages/artifact/src/artifact.ts @@ -30,17 +30,17 @@ export async function uploadArtifact( // Search for the items that will be uploaded const filesToUpload: SearchResult[] = await findFilesToUpload(name, path) + let uploadInfo: UploadInfo = { + artifactName: name, + artifactItems: [], + size: 0, + failedItems: [] + } if (filesToUpload.length === 0) { core.warning( `No files were found for the provided path: ${path}. No artifacts will be uploaded.` ) - return { - artifactName: name, - artifactItems: [], - size: 0, - failedItems: [] - } } else { /** * Create an entry for the artifact in the file container @@ -73,13 +73,11 @@ export async function uploadArtifact( `Finished uploading artifact ${name}. Reported size is ${uploadResult.size} bytes. There were ${uploadResult.failedItems.length} items that failed to upload` ) - return { - artifactName: name, - artifactItems: filesToUpload.map(item => item.absoluteFilePath), - size: uploadResult.size, - failedItems: uploadResult.failedItems - } + uploadInfo.artifactItems = filesToUpload.map(item => item.absoluteFilePath) + uploadResult.size = uploadResult.size + uploadResult.failedItems = uploadResult.failedItems } + return uploadInfo } /* diff --git a/packages/artifact/src/upload-artifact-http-client.ts b/packages/artifact/src/upload-artifact-http-client.ts index eb7f56932d..034cb476f0 100644 --- a/packages/artifact/src/upload-artifact-http-client.ts +++ b/packages/artifact/src/upload-artifact-http-client.ts @@ -277,6 +277,8 @@ async function uploadChunk( } else { // eslint-disable-next-line no-console console.log(`Unable to upload chunk even after retrying`) + // eslint-disable-next-line no-console + console.log(response) return false } } @@ -314,10 +316,13 @@ export async function patchArtifactSize( ) const body: string = await rawResponse.readBody() - if (rawResponse.message.statusCode === 200) { - const successResponse: PatchArtifactSizeSuccessResponse = JSON.parse(body) - debug(`Artifact ${artifactName} uploaded successfully, total size ${size}`) - debug(successResponse.toString()) + if ( + rawResponse.message.statusCode && + isSuccessStatusCode(rawResponse.message.statusCode) + ) { + debug( + `Artifact ${artifactName} has been successfully uploaded, total size ${size}` + ) } else if (rawResponse.message.statusCode === 404) { throw new Error(`An Artifact with the name ${artifactName} was not found`) } else { From 6f6aac1b4c900f433243f6f0b18ba7b7af7ffcde Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Mon, 3 Feb 2020 10:59:26 -0500 Subject: [PATCH 29/46] Code cleanup --- packages/artifact/src/artifact.ts | 6 +++--- packages/artifact/src/contracts.ts | 10 ---------- packages/artifact/src/upload-artifact-http-client.ts | 1 - 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/packages/artifact/src/artifact.ts b/packages/artifact/src/artifact.ts index 89314f73dc..41bfb5688f 100644 --- a/packages/artifact/src/artifact.ts +++ b/packages/artifact/src/artifact.ts @@ -30,7 +30,7 @@ export async function uploadArtifact( // Search for the items that will be uploaded const filesToUpload: SearchResult[] = await findFilesToUpload(name, path) - let uploadInfo: UploadInfo = { + const uploadInfo: UploadInfo = { artifactName: name, artifactItems: [], size: 0, @@ -74,8 +74,8 @@ export async function uploadArtifact( ) uploadInfo.artifactItems = filesToUpload.map(item => item.absoluteFilePath) - uploadResult.size = uploadResult.size - uploadResult.failedItems = uploadResult.failedItems + uploadInfo.size = uploadResult.size + uploadInfo.failedItems = uploadResult.failedItems } return uploadInfo } diff --git a/packages/artifact/src/contracts.ts b/packages/artifact/src/contracts.ts index d3ede9eb77..8574afbd7e 100644 --- a/packages/artifact/src/contracts.ts +++ b/packages/artifact/src/contracts.ts @@ -17,16 +17,6 @@ export interface PatchArtifactSize { Size: number } -export interface PatchArtifactSizeSuccessResponse { - containerId: number - size: number - signedContent: string - type: string - name: string - url: string - uploadUrl: string -} - export interface UploadResults { size: number failedItems: string[] diff --git a/packages/artifact/src/upload-artifact-http-client.ts b/packages/artifact/src/upload-artifact-http-client.ts index 034cb476f0..f6a2dc4fce 100644 --- a/packages/artifact/src/upload-artifact-http-client.ts +++ b/packages/artifact/src/upload-artifact-http-client.ts @@ -5,7 +5,6 @@ import { CreateArtifactResponse, CreateArtifactParameters, PatchArtifactSize, - PatchArtifactSizeSuccessResponse, UploadResults } from './contracts' import * as fs from 'fs' From cd5d31afb414c6af42f60883f8f4806a7c1ba640 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Mon, 3 Feb 2020 11:07:12 -0500 Subject: [PATCH 30/46] Extra debug --- packages/artifact/src/contracts.ts | 10 ++++++++++ packages/artifact/src/upload-artifact-http-client.ts | 1 + 2 files changed, 11 insertions(+) diff --git a/packages/artifact/src/contracts.ts b/packages/artifact/src/contracts.ts index 8574afbd7e..d3ede9eb77 100644 --- a/packages/artifact/src/contracts.ts +++ b/packages/artifact/src/contracts.ts @@ -17,6 +17,16 @@ export interface PatchArtifactSize { Size: number } +export interface PatchArtifactSizeSuccessResponse { + containerId: number + size: number + signedContent: string + type: string + name: string + url: string + uploadUrl: string +} + export interface UploadResults { size: number failedItems: string[] diff --git a/packages/artifact/src/upload-artifact-http-client.ts b/packages/artifact/src/upload-artifact-http-client.ts index f6a2dc4fce..4e903bf41e 100644 --- a/packages/artifact/src/upload-artifact-http-client.ts +++ b/packages/artifact/src/upload-artifact-http-client.ts @@ -322,6 +322,7 @@ export async function patchArtifactSize( debug( `Artifact ${artifactName} has been successfully uploaded, total size ${size}` ) + debug(body) } else if (rawResponse.message.statusCode === 404) { throw new Error(`An Artifact with the name ${artifactName} was not found`) } else { From 09b1c39421d8dfdcd26f0897e7cbcbd08ab45db9 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Mon, 3 Feb 2020 20:14:06 -0500 Subject: [PATCH 31/46] Move glob search out of the package --- packages/artifact/__tests__/search.test.ts | 284 -------------- .../__tests__/upload-specification.test.ts | 349 ++++++++++++++++++ packages/artifact/__tests__/upload.test.ts | 18 +- packages/artifact/package-lock.json | 36 -- packages/artifact/package.json | 1 - packages/artifact/src/artifact.ts | 47 ++- packages/artifact/src/search.ts | 99 ----- .../src/upload-artifact-http-client.ts | 4 +- packages/artifact/src/upload-specification.ts | 87 +++++ 9 files changed, 470 insertions(+), 455 deletions(-) delete mode 100644 packages/artifact/__tests__/search.test.ts create mode 100644 packages/artifact/__tests__/upload-specification.test.ts delete mode 100644 packages/artifact/src/search.ts create mode 100644 packages/artifact/src/upload-specification.ts diff --git a/packages/artifact/__tests__/search.test.ts b/packages/artifact/__tests__/search.test.ts deleted file mode 100644 index b6ad89662a..0000000000 --- a/packages/artifact/__tests__/search.test.ts +++ /dev/null @@ -1,284 +0,0 @@ -import {promises as fs} from 'fs' -import * as path from 'path' -import {findFilesToUpload} from '../src/search' -import * as io from '../../io/src/io' - -const artifactName = 'my-artifact' -const root = path.join(__dirname, '_temp', 'artifact-search') -const goodItem1Path = path.join( - root, - 'folder-a', - 'folder-b', - 'folder-c', - 'good-item1.txt' -) -const goodItem2Path = path.join(root, 'folder-d', 'good-item2.txt') -const goodItem3Path = path.join(root, 'folder-d', 'good-item3.txt') -const goodItem4Path = path.join(root, 'folder-d', 'good-item4.txt') -const goodItem5Path = path.join(root, 'good-item5.txt') -const badItem1Path = path.join( - root, - 'folder-a', - 'folder-b', - 'folder-c', - 'bad-item1.txt' -) -const badItem2Path = path.join(root, 'folder-d', 'bad-item2.txt') -const badItem3Path = path.join(root, 'folder-f', 'bad-item3.txt') -const badItem4Path = path.join(root, 'folder-h', 'folder-i', 'bad-item4.txt') -const badItem5Path = path.join(root, 'folder-h', 'folder-i', 'bad-item5.txt') -const extraFileInFolderCPath = path.join( - root, - 'folder-a', - 'folder-b', - 'folder-c', - 'extra-file-in-folder-c.txt' -) -const amazingFileInFolderHPath = path.join(root, 'folder-h', 'amazing-item.txt') - -describe('Search', () => { - beforeAll(async () => { - // clear temp directory - await io.rmRF(root) - await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-e'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-d'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-f'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-g'), { - recursive: true - }) - await fs.mkdir(path.join(root, 'folder-h', 'folder-i'), { - recursive: true - }) - - await fs.writeFile(goodItem1Path, 'good item1 file') - await fs.writeFile(goodItem2Path, 'good item2 file') - await fs.writeFile(goodItem3Path, 'good item3 file') - await fs.writeFile(goodItem4Path, 'good item4 file') - await fs.writeFile(goodItem5Path, 'good item5 file') - - await fs.writeFile(badItem1Path, 'bad item1 file') - await fs.writeFile(badItem2Path, 'bad item2 file') - await fs.writeFile(badItem3Path, 'bad item3 file') - await fs.writeFile(badItem4Path, 'bad item4 file') - await fs.writeFile(badItem5Path, 'bad item5 file') - - await fs.writeFile(extraFileInFolderCPath, 'extra file') - - await fs.writeFile(amazingFileInFolderHPath, 'amazing file') - /* - Directory structure of files that get created: - root/ - folder-a/ - folder-b/ - folder-c/ - good-item1.txt - bad-item1.txt - extra-file-in-folder-c.txt - folder-e/ - folder-d/ - good-item2.txt - good-item3.txt - good-item4.txt - bad-item2.txt - folder-f/ - bad-item3.txt - folder-g/ - folder-h/ - amazing-item.txt - folder-i/ - bad-item4.txt - bad-item5.txt - good-item5.txt - */ - }) - - /** - * Expected to find one item with full file path provided - * Only 1 item is expected to be found so the uploadFilePath is expected to be {artifactName}/extra-file-in-folder-c.txt - */ - it('Single file search - full path', async () => { - const expectedUploadFilePath = path.join( - artifactName, - 'extra-file-in-folder-c.txt' - ) - const searchResult = await findFilesToUpload( - artifactName, - extraFileInFolderCPath - ) - /* - searchResult[] should be equal to: - [ - { - absoluteFilePath: extraFileInFolderCPath - uploadFilePath: my-artifact/extra-file-in-folder-c.txt - } - ] - */ - - expect(searchResult.length).toEqual(1) - expect(searchResult[0].uploadFilePath).toEqual(expectedUploadFilePath) - expect(searchResult[0].absoluteFilePath).toEqual(extraFileInFolderCPath) - }) - - /** - * Expected to find one item with the provided wildcard pattern - * Only 1 item is expected to be found so the uploadFilePath is expected to be {artifactName}/good-item1.txt - */ - it('Single file search - wildcard pattern', async () => { - const searchPath = path.join(root, '**/good*m1.txt') - const expectedUploadFilePath = path.join(artifactName, 'good-item1.txt') - const searchResult = await findFilesToUpload(artifactName, searchPath) - /* - searchResult should be equal to: - [ - { - absoluteFilePath: goodItem1Path - uploadFilePath: my-artifact/good-item1.txt - } - ] - */ - - expect(searchResult.length).toEqual(1) - expect(searchResult[0].uploadFilePath).toEqual(expectedUploadFilePath) - expect(searchResult[0].absoluteFilePath).toEqual(goodItem1Path) - }) - - /** - * Creates a directory with multiple files and subdirectories, includes some empty directories and misc files - * Only files corresponding to the good* pattern should be found - */ - it('Wildcard search for multiple files', async () => { - const searchPath = path.join(root, '**/good*') - const searchResult = await findFilesToUpload(artifactName, searchPath) - /* - searchResult should be equal to: - [ - { - absoluteFilePath: goodItem1Path - uploadFilePath: my-artifact/folder-a/folder-b/folder-c/good-item1.txt - }, - { - absoluteFilePath: goodItem2Path - uploadFilePath: my-artifact/folder-d/good-item2.txt - }, - { - absoluteFilePath: goodItem3Path - uploadFilePath: my-artifact/folder-d/good-item3.txt - }, - { - absoluteFilePath: goodItem4Path - uploadFilePath: my-artifact/folder-d/good-item4.txt - }, - { - absoluteFilePath: goodItem5Path - uploadFilePath: my-artifact/good-item5.txt - } - ] - */ - - expect(searchResult.length).toEqual(5) - - const absolutePaths = searchResult.map(item => item.absoluteFilePath) - expect(absolutePaths.includes(goodItem1Path)).toEqual(true) - expect(absolutePaths.includes(goodItem2Path)).toEqual(true) - expect(absolutePaths.includes(goodItem3Path)).toEqual(true) - expect(absolutePaths.includes(goodItem4Path)).toEqual(true) - expect(absolutePaths.includes(goodItem5Path)).toEqual(true) - - for (const result of searchResult) { - if (result.absoluteFilePath === goodItem1Path) { - expect(result.uploadFilePath).toEqual( - path.join( - artifactName, - 'folder-a', - 'folder-b', - 'folder-c', - 'good-item1.txt' - ) - ) - } - if (result.absoluteFilePath === goodItem2Path) { - expect(result.uploadFilePath).toEqual( - path.join(artifactName, 'folder-d', 'good-item2.txt') - ) - } - if (result.absoluteFilePath === goodItem3Path) { - expect(result.uploadFilePath).toEqual( - path.join(artifactName, 'folder-d', 'good-item3.txt') - ) - } - if (result.absoluteFilePath === goodItem4Path) { - expect(result.uploadFilePath).toEqual( - path.join(artifactName, 'folder-d', 'good-item4.txt') - ) - } - if (result.absoluteFilePath === goodItem5Path) { - expect(result.uploadFilePath).toEqual( - path.join(artifactName, 'good-item5.txt') - ) - } - } - }) - - /** - * Creates a directory with multiple files and subdirectories, includes some empty directories - * All items are expected to be found - */ - it('Directory search - find everything', async () => { - const searchResult = await findFilesToUpload( - artifactName, - path.join(root, 'folder-h') - ) - /* - searchResult should be equal to: - [ - { - absoluteFilePath: amazingFileInFolderHPath - uploadFilePath: my-artifact/folder-h/amazing-item.txt - }, - { - absoluteFilePath: badItem4Path - uploadFilePath: my-artifact/folder-h/folder-i/bad-item4.txt - }, - { - absoluteFilePath: badItem5Path - uploadFilePath: my-artifact/folder-h/folder-i/bad-item5.txt - } - ] - */ - - expect(searchResult.length).toEqual(3) - - const absolutePaths = searchResult.map(item => item.absoluteFilePath) - expect(absolutePaths.includes(amazingFileInFolderHPath)).toEqual(true) - expect(absolutePaths.includes(badItem4Path)).toEqual(true) - expect(absolutePaths.includes(badItem5Path)).toEqual(true) - - for (const result of searchResult) { - if (result.absoluteFilePath === amazingFileInFolderHPath) { - expect(result.uploadFilePath).toEqual( - path.join(artifactName, 'amazing-item.txt') - ) - } - if (result.absoluteFilePath === badItem4Path) { - expect(result.uploadFilePath).toEqual( - path.join(artifactName, 'folder-i', 'bad-item4.txt') - ) - } - if (result.absoluteFilePath === badItem5Path) { - expect(result.uploadFilePath).toEqual( - path.join(artifactName, 'folder-i', 'bad-item5.txt') - ) - } - } - }) -}) diff --git a/packages/artifact/__tests__/upload-specification.test.ts b/packages/artifact/__tests__/upload-specification.test.ts new file mode 100644 index 0000000000..64942c3a66 --- /dev/null +++ b/packages/artifact/__tests__/upload-specification.test.ts @@ -0,0 +1,349 @@ +import {promises as fs} from 'fs' +import * as path from 'path' +import {getUploadSpecification} from '../src/upload-specification' +import * as io from '../../io/src/io' + +const artifactName = 'my-artifact' +const root = path.join(__dirname, '_temp', 'upload-specification') +const goodItem1Path = path.join( + root, + 'folder-a', + 'folder-b', + 'folder-c', + 'good-item1.txt' +) +const goodItem2Path = path.join(root, 'folder-d', 'good-item2.txt') +const goodItem3Path = path.join(root, 'folder-d', 'good-item3.txt') +const goodItem4Path = path.join(root, 'folder-d', 'good-item4.txt') +const goodItem5Path = path.join(root, 'good-item5.txt') +const badItem1Path = path.join( + root, + 'folder-a', + 'folder-b', + 'folder-c', + 'bad-item1.txt' +) +const badItem2Path = path.join(root, 'folder-d', 'bad-item2.txt') +const badItem3Path = path.join(root, 'folder-f', 'bad-item3.txt') +const badItem4Path = path.join(root, 'folder-h', 'folder-i', 'bad-item4.txt') +const badItem5Path = path.join(root, 'folder-h', 'folder-i', 'bad-item5.txt') +const extraFileInFolderCPath = path.join( + root, + 'folder-a', + 'folder-b', + 'folder-c', + 'extra-file-in-folder-c.txt' +) +const amazingFileInFolderHPath = path.join(root, 'folder-h', 'amazing-item.txt') + +const artifactFilesToUpload = [ + goodItem1Path, + goodItem2Path, + goodItem3Path, + goodItem4Path, + goodItem5Path, + extraFileInFolderCPath, + amazingFileInFolderHPath +] + +describe('Search', () => { + beforeAll(async () => { + // clear temp directory + await io.rmRF(root) + await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-e'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-d'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-f'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-g'), { + recursive: true + }) + await fs.mkdir(path.join(root, 'folder-h', 'folder-i'), { + recursive: true + }) + + await fs.writeFile(goodItem1Path, 'good item1 file') + await fs.writeFile(goodItem2Path, 'good item2 file') + await fs.writeFile(goodItem3Path, 'good item3 file') + await fs.writeFile(goodItem4Path, 'good item4 file') + await fs.writeFile(goodItem5Path, 'good item5 file') + + await fs.writeFile(badItem1Path, 'bad item1 file') + await fs.writeFile(badItem2Path, 'bad item2 file') + await fs.writeFile(badItem3Path, 'bad item3 file') + await fs.writeFile(badItem4Path, 'bad item4 file') + await fs.writeFile(badItem5Path, 'bad item5 file') + + await fs.writeFile(extraFileInFolderCPath, 'extra file') + + await fs.writeFile(amazingFileInFolderHPath, 'amazing file') + /* + Directory structure of files that get created: + root/ + folder-a/ + folder-b/ + folder-c/ + good-item1.txt + bad-item1.txt + extra-file-in-folder-c.txt + folder-e/ + folder-d/ + good-item2.txt + good-item3.txt + good-item4.txt + bad-item2.txt + folder-f/ + bad-item3.txt + folder-g/ + folder-h/ + amazing-item.txt + folder-i/ + bad-item4.txt + bad-item5.txt + good-item5.txt + */ + }) + + it('Upload Specification - Fail non-existent rootDirectory', async () => { + const invalidRootDirectory = path.join( + __dirname, + '_temp', + 'upload-specification-invalid' + ) + expect(() => { + getUploadSpecification( + artifactName, + invalidRootDirectory, + artifactFilesToUpload + ) + }).toThrow(`Provided rootDirectory ${invalidRootDirectory} does not exist`) + }) + + it('Upload Specification - Fail invalid rootDirectory', async () => { + expect(() => { + getUploadSpecification(artifactName, goodItem1Path, artifactFilesToUpload) + }).toThrow( + `Provided rootDirectory ${goodItem1Path} is not a valid directory` + ) + }) + + it('Upload Specification - File does not exist', async () => { + const fakeFilePath = path.join( + artifactName, + 'folder-a', + 'folder-b', + 'non-existent-file.txt' + ) + expect(() => { + getUploadSpecification(artifactName, root, [fakeFilePath]) + }).toThrow(`File ${fakeFilePath} does not exist`) + }) + + it('Upload Specification - Non parent directory', async () => { + const folderADirectory = path.join(root, 'folder-a') + const artifactFiles = [ + goodItem1Path, + badItem1Path, + extraFileInFolderCPath, + goodItem5Path + ] + expect(() => { + getUploadSpecification(artifactName, folderADirectory, artifactFiles) + }).toThrow( + `The rootDirectory: ${folderADirectory} is not a parent directory of the file: ${goodItem5Path}` + ) + }) + + it('Upload Specification - Success', async () => { + const specifications = getUploadSpecification( + artifactName, + root, + artifactFilesToUpload + ) + expect(specifications.length).toEqual(7) + + const absolutePaths = specifications.map(item => item.absoluteFilePath) + expect(absolutePaths.includes(goodItem1Path)).toEqual(true) + expect(absolutePaths.includes(goodItem2Path)).toEqual(true) + expect(absolutePaths.includes(goodItem3Path)).toEqual(true) + expect(absolutePaths.includes(goodItem4Path)).toEqual(true) + expect(absolutePaths.includes(goodItem5Path)).toEqual(true) + expect(absolutePaths.includes(extraFileInFolderCPath)).toEqual(true) + expect(absolutePaths.includes(amazingFileInFolderHPath)).toEqual(true) + + for (const specification of specifications) { + if (specification.absoluteFilePath === goodItem1Path) { + expect(specification.uploadFilePath).toEqual( + path.join( + artifactName, + 'folder-a', + 'folder-b', + 'folder-c', + 'good-item1.txt' + ) + ) + } + if (specification.absoluteFilePath === goodItem2Path) { + expect(specification.uploadFilePath).toEqual( + path.join(artifactName, 'folder-d', 'good-item2.txt') + ) + } + if (specification.absoluteFilePath === goodItem3Path) { + expect(specification.uploadFilePath).toEqual( + path.join(artifactName, 'folder-d', 'good-item3.txt') + ) + } + if (specification.absoluteFilePath === goodItem4Path) { + expect(specification.uploadFilePath).toEqual( + path.join(artifactName, 'folder-d', 'good-item4.txt') + ) + } + if (specification.absoluteFilePath === goodItem5Path) { + expect(specification.uploadFilePath).toEqual( + path.join(artifactName, 'good-item5.txt') + ) + } + if (specification.absoluteFilePath === extraFileInFolderCPath) { + expect(specification.uploadFilePath).toEqual( + path.join( + artifactName, + 'folder-a', + 'folder-b', + 'folder-c', + 'extra-file-in-folder-c.txt' + ) + ) + } + if (specification.absoluteFilePath === amazingFileInFolderHPath) { + expect(specification.uploadFilePath).toEqual( + path.join(artifactName, 'folder-h', 'amazing-item.txt') + ) + } + } + }) + + it('Upload Specification - Success with extra slash', async () => { + const rootWithSlash = `${root}/` + const specifications = getUploadSpecification( + artifactName, + rootWithSlash, + artifactFilesToUpload + ) + expect(specifications.length).toEqual(7) + + const absolutePaths = specifications.map(item => item.absoluteFilePath) + expect(absolutePaths.includes(goodItem1Path)).toEqual(true) + expect(absolutePaths.includes(goodItem2Path)).toEqual(true) + expect(absolutePaths.includes(goodItem3Path)).toEqual(true) + expect(absolutePaths.includes(goodItem4Path)).toEqual(true) + expect(absolutePaths.includes(goodItem5Path)).toEqual(true) + expect(absolutePaths.includes(extraFileInFolderCPath)).toEqual(true) + expect(absolutePaths.includes(amazingFileInFolderHPath)).toEqual(true) + + for (const specification of specifications) { + if (specification.absoluteFilePath === goodItem1Path) { + expect(specification.uploadFilePath).toEqual( + path.join( + artifactName, + 'folder-a', + 'folder-b', + 'folder-c', + 'good-item1.txt' + ) + ) + } + if (specification.absoluteFilePath === goodItem2Path) { + expect(specification.uploadFilePath).toEqual( + path.join(artifactName, 'folder-d', 'good-item2.txt') + ) + } + if (specification.absoluteFilePath === goodItem3Path) { + expect(specification.uploadFilePath).toEqual( + path.join(artifactName, 'folder-d', 'good-item3.txt') + ) + } + if (specification.absoluteFilePath === goodItem4Path) { + expect(specification.uploadFilePath).toEqual( + path.join(artifactName, 'folder-d', 'good-item4.txt') + ) + } + if (specification.absoluteFilePath === goodItem5Path) { + expect(specification.uploadFilePath).toEqual( + path.join(artifactName, 'good-item5.txt') + ) + } + if (specification.absoluteFilePath === extraFileInFolderCPath) { + expect(specification.uploadFilePath).toEqual( + path.join( + artifactName, + 'folder-a', + 'folder-b', + 'folder-c', + 'extra-file-in-folder-c.txt' + ) + ) + } + if (specification.absoluteFilePath === amazingFileInFolderHPath) { + expect(specification.uploadFilePath).toEqual( + path.join(artifactName, 'folder-h', 'amazing-item.txt') + ) + } + } + }) + + it('Upload Specification - Directories should not be included', async () => { + const folderEPath = path.join(root, 'folder-a', 'folder-b', 'folder-e') + const filesWithDirectory = [ + goodItem1Path, + goodItem4Path, + folderEPath, + badItem3Path + ] + const specifications = getUploadSpecification( + artifactName, + root, + filesWithDirectory + ) + expect(specifications.length).toEqual(3) + const absolutePaths = specifications.map(item => item.absoluteFilePath) + expect(absolutePaths.includes(goodItem1Path)).toEqual(true) + expect(absolutePaths.includes(goodItem4Path)).toEqual(true) + expect(absolutePaths.includes(badItem3Path)).toEqual(true) + + for (const specification of specifications) { + if (specification.absoluteFilePath === goodItem1Path) { + expect(specification.uploadFilePath).toEqual( + path.join( + artifactName, + 'folder-a', + 'folder-b', + 'folder-c', + 'good-item1.txt' + ) + ) + } + if (specification.absoluteFilePath === goodItem2Path) { + expect(specification.uploadFilePath).toEqual( + path.join(artifactName, 'folder-d', 'good-item2.txt') + ) + } + if (specification.absoluteFilePath === goodItem4Path) { + expect(specification.uploadFilePath).toEqual( + path.join(artifactName, 'folder-d', 'good-item4.txt') + ) + } + if (specification.absoluteFilePath === badItem3Path) { + expect(specification.uploadFilePath).toEqual( + path.join(artifactName, 'folder-f', 'bad-item3.txt') + ) + } + } + }) +}) diff --git a/packages/artifact/__tests__/upload.test.ts b/packages/artifact/__tests__/upload.test.ts index 531e330d01..b96eff7c86 100644 --- a/packages/artifact/__tests__/upload.test.ts +++ b/packages/artifact/__tests__/upload.test.ts @@ -10,7 +10,7 @@ import { CreateArtifactResponse, PatchArtifactSizeSuccessResponse } from '../src/contracts' -import {SearchResult} from '../src/search' +import {UploadSpecification} from '../src/upload-specification' const root = path.join(__dirname, '_temp', 'artifact-upload') const file1Path = path.join(root, 'file1.txt') @@ -104,7 +104,7 @@ describe('Upload Tests', () => { * focuses solely on the upload APIs so searchResult[] will be hard-coded */ const artifactName = 'successful-artifact' - const searchResult: SearchResult[] = [ + const uploadSpecification: UploadSpecification[] = [ { absoluteFilePath: file1Path, uploadFilePath: `${artifactName}/file1.txt` @@ -132,14 +132,14 @@ describe('Upload Tests', () => { const uploadUrl = `${getRuntimeUrl()}_apis/resources/Containers/13` const uploadResult = await uploadHttpClient.uploadArtifactToFileContainer( uploadUrl, - searchResult + uploadSpecification ) expect(uploadResult.failedItems.length).toEqual(0) expect(uploadResult.size).toEqual(expectedTotalSize) }) it('Upload Artifact - Failed Single File Upload', async () => { - const searchResult: SearchResult[] = [ + const uploadSpecification: UploadSpecification[] = [ { absoluteFilePath: file1Path, uploadFilePath: `this-file-upload-will-fail` @@ -149,7 +149,7 @@ describe('Upload Tests', () => { const uploadUrl = `${getRuntimeUrl()}_apis/resources/Containers/13` const uploadResult = await uploadHttpClient.uploadArtifactToFileContainer( uploadUrl, - searchResult + uploadSpecification ) expect(uploadResult.failedItems.length).toEqual(1) expect(uploadResult.size).toEqual(0) @@ -157,7 +157,7 @@ describe('Upload Tests', () => { it('Upload Artifact - Partial Upload Continue On Error', async () => { const artifactName = 'partial-artifact' - const searchResult: SearchResult[] = [ + const uploadSpecification: UploadSpecification[] = [ { absoluteFilePath: file1Path, uploadFilePath: `${artifactName}/file1.txt` @@ -184,7 +184,7 @@ describe('Upload Tests', () => { const uploadUrl = `${getRuntimeUrl()}_apis/resources/Containers/13` const uploadResult = await uploadHttpClient.uploadArtifactToFileContainer( uploadUrl, - searchResult, + uploadSpecification, {continueOnError: true} ) expect(uploadResult.failedItems.length).toEqual(1) @@ -193,7 +193,7 @@ describe('Upload Tests', () => { it('Upload Artifact - Partial Upload Fail Fast', async () => { const artifactName = 'partial-artifact' - const searchResult: SearchResult[] = [ + const uploadSpecification: UploadSpecification[] = [ { absoluteFilePath: file1Path, uploadFilePath: `${artifactName}/file1.txt` @@ -220,7 +220,7 @@ describe('Upload Tests', () => { const uploadUrl = `${getRuntimeUrl()}_apis/resources/Containers/13` const uploadResult = await uploadHttpClient.uploadArtifactToFileContainer( uploadUrl, - searchResult, + uploadSpecification, {continueOnError: false} ) expect(uploadResult.failedItems.length).toEqual(2) diff --git a/packages/artifact/package-lock.json b/packages/artifact/package-lock.json index 3c5cd5d048..8e26600de7 100644 --- a/packages/artifact/package-lock.json +++ b/packages/artifact/package-lock.json @@ -9,46 +9,10 @@ "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.2.1.tgz", "integrity": "sha512-xD+CQd9p4lU7ZfRqmUcbJpqR+Ss51rJRVeXMyOLrZQImN9/8Sy/BEUBnHO/UKD3z03R686PVTLfEPmkropGuLw==" }, - "@actions/glob": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/@actions/glob/-/glob-0.1.0.tgz", - "integrity": "sha512-lx8SzyQ2FE9+UUvjqY1f28QbTJv+w8qP7kHHbfQRhphrlcx0Mdmm1tZdGJzfxv1jxREa/sLW4Oy8CbGQKCJySA==", - "requires": { - "@actions/core": "^1.2.0", - "minimatch": "^3.0.4" - } - }, "@actions/http-client": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-1.0.1.tgz", "integrity": "sha512-vy5DhqTJ1gtEkpRrD/6BHhUlkeyccrOX0BT9KmtO5TWxe5KSSwVHFE+J15Z0dG+tJwZJ/nHC4slUIyqpkahoMg==" - }, - "balanced-match": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" - }, - "brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "requires": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" - }, - "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "requires": { - "brace-expansion": "^1.1.7" - } } } } diff --git a/packages/artifact/package.json b/packages/artifact/package.json index 765d543bf8..c99b800b1a 100644 --- a/packages/artifact/package.json +++ b/packages/artifact/package.json @@ -37,7 +37,6 @@ }, "dependencies": { "@actions/core": "^1.2.1", - "@actions/glob": "^0.1.0", "@actions/http-client": "^1.0.1" } } diff --git a/packages/artifact/src/artifact.ts b/packages/artifact/src/artifact.ts index 41bfb5688f..bdf002db0c 100644 --- a/packages/artifact/src/artifact.ts +++ b/packages/artifact/src/artifact.ts @@ -1,5 +1,8 @@ import * as core from '@actions/core' -import {SearchResult, findFilesToUpload} from './search' +import { + UploadSpecification, + getUploadSpecification +} from './upload-specification' import { createArtifactInFileContainer, uploadArtifactToFileContainer, @@ -13,23 +16,25 @@ import {checkArtifactName} from './utils' * Uploads an artifact * * @param name the name of the artifact, required - * @param path the directory, file, or glob pattern to denote what will be uploaded, required + * @param files a list of absolute paths that denote what files should be uploaded + * @param rootDirectory an absolute file path that denotes the root parent directory of the files being uploaded * @param options extra options for customizing the upload behavior * @returns single UploadInfo object */ export async function uploadArtifact( name: string, - path: string, + files: string[], + rootDirectory: string, options?: UploadOptions ): Promise { checkArtifactName(name) - if (!path) { - throw new Error('Upload path must be provided') - } - - // Search for the items that will be uploaded - const filesToUpload: SearchResult[] = await findFilesToUpload(name, path) + // Get specification for the files being uploaded + const uploadSpecification: UploadSpecification[] = getUploadSpecification( + name, + rootDirectory, + files + ) const uploadInfo: UploadInfo = { artifactName: name, artifactItems: [], @@ -37,14 +42,10 @@ export async function uploadArtifact( failedItems: [] } - if (filesToUpload.length === 0) { - core.warning( - `No files were found for the provided path: ${path}. No artifacts will be uploaded.` - ) + if (uploadSpecification.length === 0) { + core.warning(`No files found that can be uploaded`) } else { - /** - * Create an entry for the artifact in the file container - */ + // Create an entry for the artifact in the file container const response = await createArtifactInFileContainer(name) if (!response.fileContainerResourceUrl) { core.debug(response.toString()) @@ -54,18 +55,14 @@ export async function uploadArtifact( } core.debug(`Upload Resource URL: ${response.fileContainerResourceUrl}`) - /** - * Upload each of the files that were found concurrently - */ + // Upload each of the files that were found concurrently const uploadResult = await uploadArtifactToFileContainer( response.fileContainerResourceUrl, - filesToUpload, + uploadSpecification, options ) - /** - * Update the size of the artifact to indicate we are done uploading - */ + //Update the size of the artifact to indicate we are done uploading await patchArtifactSize(uploadResult.size, name) // eslint-disable-next-line no-console @@ -73,7 +70,9 @@ export async function uploadArtifact( `Finished uploading artifact ${name}. Reported size is ${uploadResult.size} bytes. There were ${uploadResult.failedItems.length} items that failed to upload` ) - uploadInfo.artifactItems = filesToUpload.map(item => item.absoluteFilePath) + uploadInfo.artifactItems = uploadSpecification.map( + item => item.absoluteFilePath + ) uploadInfo.size = uploadResult.size uploadInfo.failedItems = uploadResult.failedItems } diff --git a/packages/artifact/src/search.ts b/packages/artifact/src/search.ts deleted file mode 100644 index c0a37ca1ba..0000000000 --- a/packages/artifact/src/search.ts +++ /dev/null @@ -1,99 +0,0 @@ -import {debug} from '@actions/core' -import * as glob from '@actions/glob' -import {lstatSync} from 'fs' -import {join, basename} from 'path' - -export interface SearchResult { - absoluteFilePath: string - uploadFilePath: string -} - -/** - * Searches the provided path and returns the files that will be uploaded as part of the artifact - * @param {string} searchPath Wildcard pattern, directory or individual file that is provided by the user to specify what should be uploaded - * @param {string} artifactName The name of the artifact, used as the root directory when uploading artifacts to glob storage - * @return A list of files that should be uploaded along with the paths they will be uploaded with - */ -export async function findFilesToUpload( - artifactName: string, - searchPath: string -): Promise { - const searchResults: SearchResult[] = [] - const itemsToUpload: string[] = [] - const options: glob.GlobOptions = { - followSymbolicLinks: true, - implicitDescendants: true, - omitBrokenSymbolicLinks: true - } - const globber = await glob.create(searchPath, options) - - const rawSearchResults: string[] = await globber.glob() - /** - * Directories will be rejected if attempted to be uploaded. This includes just empty - * directories so filter any directories out from the raw search results - */ - for (const searchResult of rawSearchResults) { - if (!lstatSync(searchResult).isDirectory()) { - itemsToUpload.push(searchResult) - } else { - debug( - `Removing ${searchResult} from rawSearchResults because it is a directory` - ) - } - } - - if (itemsToUpload.length === 0) { - return searchResults - } - // eslint-disable-next-line no-console - console.log( - `Found the following ${itemsToUpload.length} items that will be uploaded as part of the artifact:` - ) - // eslint-disable-next-line no-console - console.log(itemsToUpload) - - /** - * Only a single search pattern is being included so only 1 searchResult is expected. In the future if multiple search patterns are - * simultaneously supported this will change - */ - const searchPaths: string[] = globber.getSearchPaths() - if (searchResults.length > 1) { - // eslint-disable-next-line no-console - console.log(searchResults) - throw new Error('Only 1 search path should be returned') - } - - /** - * Creates the path that the artifact will be uploaded with. The artifact name will always be the root directory so that - * it can be distinguished from other artifacts that are uploaded to the same file Container/glob storage during a run - */ - if (itemsToUpload.length === 1) { - // A single artifact will be uploaded, the upload path will always be in the form ${artifactName}\${singleArtifactName} - searchResults.push({ - absoluteFilePath: itemsToUpload[0], - uploadFilePath: join(artifactName, basename(itemsToUpload[0])) - }) - } else { - /** - * multiple files will be uploaded as part of the artifact - * The search path denotes the base path that was used to find the file. It will be removed from the absolute path and - * the artifact name will be prepended to create the path used during upload - */ - for (const uploadItem of itemsToUpload) { - const uploadPath: string = uploadItem.replace(searchPaths[0], '') - searchResults.push({ - absoluteFilePath: uploadItem, - uploadFilePath: artifactName.concat(uploadPath) - }) - } - } - - debug('SearchResult includes the following information:') - for (const searchResult of searchResults) { - debug( - `Absolute File Path: ${searchResult.absoluteFilePath}\nUpload file path: ${searchResult.uploadFilePath}` - ) - } - - return searchResults -} diff --git a/packages/artifact/src/upload-artifact-http-client.ts b/packages/artifact/src/upload-artifact-http-client.ts index 4e903bf41e..13468a397b 100644 --- a/packages/artifact/src/upload-artifact-http-client.ts +++ b/packages/artifact/src/upload-artifact-http-client.ts @@ -8,7 +8,7 @@ import { UploadResults } from './contracts' import * as fs from 'fs' -import {SearchResult} from './search' +import {UploadSpecification} from './upload-specification' import {UploadOptions} from './upload-options' import {URL} from 'url' import { @@ -71,7 +71,7 @@ export async function createArtifactInFileContainer( */ export async function uploadArtifactToFileContainer( uploadUrl: string, - filesToUpload: SearchResult[], + filesToUpload: UploadSpecification[], options?: UploadOptions ): Promise { const client = createHttpClient(getRuntimeToken()) diff --git a/packages/artifact/src/upload-specification.ts b/packages/artifact/src/upload-specification.ts new file mode 100644 index 0000000000..40456cdc03 --- /dev/null +++ b/packages/artifact/src/upload-specification.ts @@ -0,0 +1,87 @@ +import * as fs from 'fs' +import {debug} from '@actions/core' +import {join, normalize} from 'path' +import {checkArtifactName} from './utils' + +export interface UploadSpecification { + absoluteFilePath: string + uploadFilePath: string +} + +/** + * Creates a specification that describes how each file that is part of the artifact will be uploaded + * @param artifactName the name of the artifact being uploaded. Used during upload to denote where the artifact is stored on the server + * @param rootDirectory an absolute file path that denotes the path that should be removed from the beginning of each artifact file + * @param artifactFiles a list of absolute file paths that denote what should be uploaded as part of the artifact + */ +export function getUploadSpecification( + artifactName: string, + rootDirectory: string, + artifactFiles: string[] +): UploadSpecification[] { + checkArtifactName(artifactName) + + const specifications: UploadSpecification[] = [] + + if (!fs.existsSync(rootDirectory)) { + throw new Error(`Provided rootDirectory ${rootDirectory} does not exist`) + } + if (!fs.lstatSync(rootDirectory).isDirectory()) { + throw new Error( + `Provided rootDirectory ${rootDirectory} is not a valid directory` + ) + } + rootDirectory = normalize(rootDirectory) + /* + Example to demonstrate behavior + + Input: + artifactName: my-artifact + rootDirectory: '/home/user/files/plz-upload' + artifactFiles: [ + '/home/user/files/plz-upload/file1.txt', + '/home/user/files/plz-upload/file2.txt', + '/home/user/files/plz-upload/dir/file3.txt' + ] + + Output: + specifications: [ + ['/home/user/files/plz-upload/file1.txt', 'my-artifact/file1.txt'], + ['/home/user/files/plz-upload/file1.txt', 'my-artifact/file2.txt'], + ['/home/user/files/plz-upload/file1.txt', 'my-artifact/dir/file3.txt'] + ] + */ + for (let file of artifactFiles) { + if (!fs.existsSync(file)) { + throw new Error(`File ${file} does not exist`) + } + + if (!fs.lstatSync(file).isDirectory()) { + file = normalize(file) + if (!file.startsWith(rootDirectory)) { + throw new Error( + `The rootDirectory: ${rootDirectory} is not a parent directory of the file: ${file}` + ) + } + + /* + uploadFilePath denotes where the file will be uploaded in the file container on the server. During a run, if multiple artifacts are uploaded, they will all + be saved in the same container. The artifact name is used as the root directory in the container to separate and distinguish uploaded artifacts + + path.join handles all the following cases and would return 'artifact-name/file-to-upload.txt + join('artifact-name/', 'file-to-upload.txt') + join('artifact-name/', '/file-to-upload.txt') + join('artifact-name', 'file-to-upload.txt') + join('artifact-name', '/file-to-upload.txt') + */ + specifications.push({ + absoluteFilePath: file, + uploadFilePath: join(artifactName, file.replace(rootDirectory, '')) + }) + } else { + // Directories are rejected by the server during upload + debug(`Removing ${file} from rawSearchResults because it is a directory`) + } + } + return specifications +} From 049110feaea7c0df32879824bb1782965c2d52aa Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Tue, 4 Feb 2020 18:01:42 -0500 Subject: [PATCH 32/46] Code cleanup --- packages/artifact/__tests__/util.test.ts | 10 +++++----- .../artifact/src/upload-artifact-http-client.ts | 15 +++++---------- packages/artifact/src/utils.ts | 15 +++++++++++---- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/packages/artifact/__tests__/util.test.ts b/packages/artifact/__tests__/util.test.ts index bcde791d21..68d2577d5e 100644 --- a/packages/artifact/__tests__/util.test.ts +++ b/packages/artifact/__tests__/util.test.ts @@ -1,8 +1,8 @@ import * as utils from '../src/utils' import {HttpCodes} from '@actions/http-client' +import {getRuntimeUrl, getWorkFlowRunId} from '../src/config-variables' -// use the actual implementation of for these tests @actions/http-client -jest.unmock('@actions/http-client') +jest.mock('../src/config-variables') describe('Utils', () => { it('Check Artifact Name for any invalid characters', () => { @@ -38,9 +38,9 @@ describe('Utils', () => { }) it('Test constructing artifact URL', () => { - const runtimeUrl = 'https://pipelines.actions.githubusercontent.com/abcd/' - const runId = '15' - const artifactUrl = utils.getArtifactUrl(runtimeUrl, runId) + const runtimeUrl = getRuntimeUrl() + const runId = getWorkFlowRunId() + const artifactUrl = utils.getArtifactUrl() expect(artifactUrl).toEqual( `${runtimeUrl}_apis/pipelines/workflows/${runId}/artifacts?api-version=${utils.getApiVersion()}` ) diff --git a/packages/artifact/src/upload-artifact-http-client.ts b/packages/artifact/src/upload-artifact-http-client.ts index 13468a397b..7fa50cbea0 100644 --- a/packages/artifact/src/upload-artifact-http-client.ts +++ b/packages/artifact/src/upload-artifact-http-client.ts @@ -20,9 +20,6 @@ import { isSuccessStatusCode } from './utils' import { - getRuntimeToken, - getRuntimeUrl, - getWorkFlowRunId, getUploadChunkConcurrency, getUploadChunkSize, getUploadFileConcurrency @@ -41,8 +38,8 @@ export async function createArtifactInFileContainer( Name: artifactName } const data: string = JSON.stringify(parameters, null, 2) - const artifactUrl = getArtifactUrl(getRuntimeUrl(), getWorkFlowRunId()) - const client = createHttpClient(getRuntimeToken()) + const artifactUrl = getArtifactUrl() + const client = createHttpClient() const requestOptions = getRequestOptions('application/json') const rawResponse = await client.post(artifactUrl, data, requestOptions) @@ -74,7 +71,7 @@ export async function uploadArtifactToFileContainer( filesToUpload: UploadSpecification[], options?: UploadOptions ): Promise { - const client = createHttpClient(getRuntimeToken()) + const client = createHttpClient() const FILE_CONCURRENCY = getUploadFileConcurrency() const CHUNK_CONCURRENCY = getUploadChunkConcurrency() const MAX_CHUNK_SIZE = getUploadChunkSize() @@ -297,11 +294,9 @@ export async function patchArtifactSize( size: number, artifactName: string ): Promise { - const client = createHttpClient(getRuntimeToken()) + const client = createHttpClient() const requestOptions = getRequestOptions('application/json') - const resourceUrl = new URL( - getArtifactUrl(getRuntimeUrl(), getWorkFlowRunId()) - ) + const resourceUrl = new URL(getArtifactUrl()) resourceUrl.searchParams.append('artifactName', artifactName) const parameters: PatchArtifactSize = {Size: size} diff --git a/packages/artifact/src/utils.ts b/packages/artifact/src/utils.ts index 8d06ba6bbf..e706b4aabf 100644 --- a/packages/artifact/src/utils.ts +++ b/packages/artifact/src/utils.ts @@ -2,6 +2,11 @@ import {debug} from '@actions/core' import {HttpCodes, HttpClient} from '@actions/http-client' import {BearerCredentialHandler} from '@actions/http-client/auth' import {IHeaders} from '@actions/http-client/interfaces' +import { + getRuntimeToken, + getRuntimeUrl, + getWorkFlowRunId +} from './config-variables' /** * Parses a env variable that is a number @@ -66,12 +71,14 @@ export function getRequestOptions( return requestOptions } -export function createHttpClient(token: string): HttpClient { - return new HttpClient('action/artifact', [new BearerCredentialHandler(token)]) +export function createHttpClient(): HttpClient { + return new HttpClient('action/artifact', [ + new BearerCredentialHandler(getRuntimeToken()) + ]) } -export function getArtifactUrl(runtimeUrl: string, runId: string): string { - const artifactUrl = `${runtimeUrl}_apis/pipelines/workflows/${runId}/artifacts?api-version=${getApiVersion()}` +export function getArtifactUrl(): string { + const artifactUrl = `${getRuntimeUrl()}_apis/pipelines/workflows/${getWorkFlowRunId()}/artifacts?api-version=${getApiVersion()}` debug(`Artifact Url: ${artifactUrl}`) return artifactUrl } From abd642c4c34123ab469e034ea2b43ff4b6edaaab Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Tue, 4 Feb 2020 18:42:56 -0500 Subject: [PATCH 33/46] Rename some thing and cleanup --- packages/artifact/__tests__/upload.test.ts | 6 +++--- packages/artifact/src/artifact.ts | 4 ++-- packages/artifact/src/contracts.ts | 2 +- ...upload-artifact-http-client.ts => upload-http-client.ts} | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) rename packages/artifact/src/{upload-artifact-http-client.ts => upload-http-client.ts} (99%) diff --git a/packages/artifact/__tests__/upload.test.ts b/packages/artifact/__tests__/upload.test.ts index b96eff7c86..7a21ad5af6 100644 --- a/packages/artifact/__tests__/upload.test.ts +++ b/packages/artifact/__tests__/upload.test.ts @@ -2,12 +2,12 @@ import * as http from 'http' import * as io from '../../io/src/io' import * as net from 'net' import * as path from 'path' -import * as uploadHttpClient from '../src/upload-artifact-http-client' +import * as uploadHttpClient from '../src/upload-http-client' import {promises as fs} from 'fs' import {getRuntimeUrl} from '../src/config-variables' import {HttpClient, HttpClientResponse} from '@actions/http-client' import { - CreateArtifactResponse, + ArtifactResponse, PatchArtifactSizeSuccessResponse } from '../src/contracts' import {UploadSpecification} from '../src/upload-specification' @@ -278,7 +278,7 @@ describe('Upload Tests', () => { mockMessage.statusCode = 400 } else { mockMessage.statusCode = 201 - const response: CreateArtifactResponse = { + const response: ArtifactResponse = { containerId: '13', size: -1, signedContent: 'false', diff --git a/packages/artifact/src/artifact.ts b/packages/artifact/src/artifact.ts index bdf002db0c..4b64672bc1 100644 --- a/packages/artifact/src/artifact.ts +++ b/packages/artifact/src/artifact.ts @@ -7,7 +7,7 @@ import { createArtifactInFileContainer, uploadArtifactToFileContainer, patchArtifactSize -} from './upload-artifact-http-client' +} from './upload-http-client' import {UploadInfo} from './upload-info' import {UploadOptions} from './upload-options' import {checkArtifactName} from './utils' @@ -88,7 +88,7 @@ export async function downloadArtifact( options?: DownloadOptions ): Promise { - TODO + TODO } Downloads all artifacts associated with a run. Because there are multiple artifacts being downloaded, a folder will be created for each one in the specified or default directory diff --git a/packages/artifact/src/contracts.ts b/packages/artifact/src/contracts.ts index d3ede9eb77..8124add4a7 100644 --- a/packages/artifact/src/contracts.ts +++ b/packages/artifact/src/contracts.ts @@ -1,4 +1,4 @@ -export interface CreateArtifactResponse { +export interface ArtifactResponse { containerId: string size: number signedContent: string diff --git a/packages/artifact/src/upload-artifact-http-client.ts b/packages/artifact/src/upload-http-client.ts similarity index 99% rename from packages/artifact/src/upload-artifact-http-client.ts rename to packages/artifact/src/upload-http-client.ts index 7fa50cbea0..dafae76a7a 100644 --- a/packages/artifact/src/upload-artifact-http-client.ts +++ b/packages/artifact/src/upload-http-client.ts @@ -2,7 +2,7 @@ import {debug} from '@actions/core' import {HttpClientResponse, HttpClient} from '@actions/http-client/index' import {IHttpClientResponse} from '@actions/http-client/interfaces' import { - CreateArtifactResponse, + ArtifactResponse, CreateArtifactParameters, PatchArtifactSize, UploadResults @@ -32,7 +32,7 @@ import { */ export async function createArtifactInFileContainer( artifactName: string -): Promise { +): Promise { const parameters: CreateArtifactParameters = { Type: 'actions_storage', Name: artifactName From 813bcf1e6d14ed1aabe65649e269201f3c40ddaa Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Thu, 6 Feb 2020 12:22:48 -0500 Subject: [PATCH 34/46] Rename uploadInfo to uploadResponse --- packages/artifact/src/artifact.ts | 18 +++++++++--------- .../{download-info.ts => download-response.ts} | 2 +- .../src/{upload-info.ts => upload-response.ts} | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) rename packages/artifact/src/{download-info.ts => download-response.ts} (83%) rename packages/artifact/src/{upload-info.ts => upload-response.ts} (94%) diff --git a/packages/artifact/src/artifact.ts b/packages/artifact/src/artifact.ts index 4b64672bc1..5a40016f23 100644 --- a/packages/artifact/src/artifact.ts +++ b/packages/artifact/src/artifact.ts @@ -8,7 +8,7 @@ import { uploadArtifactToFileContainer, patchArtifactSize } from './upload-http-client' -import {UploadInfo} from './upload-info' +import {UploadResponse} from './upload-response' import {UploadOptions} from './upload-options' import {checkArtifactName} from './utils' @@ -26,7 +26,7 @@ export async function uploadArtifact( files: string[], rootDirectory: string, options?: UploadOptions -): Promise { +): Promise { checkArtifactName(name) // Get specification for the files being uploaded @@ -35,7 +35,7 @@ export async function uploadArtifact( rootDirectory, files ) - const uploadInfo: UploadInfo = { + const uploadResponse: UploadResponse = { artifactName: name, artifactItems: [], size: 0, @@ -70,13 +70,13 @@ export async function uploadArtifact( `Finished uploading artifact ${name}. Reported size is ${uploadResult.size} bytes. There were ${uploadResult.failedItems.length} items that failed to upload` ) - uploadInfo.artifactItems = uploadSpecification.map( + uploadResponse.artifactItems = uploadSpecification.map( item => item.absoluteFilePath ) - uploadInfo.size = uploadResult.size - uploadInfo.failedItems = uploadResult.failedItems + uploadResponse.size = uploadResult.size + uploadResponse.failedItems = uploadResult.failedItems } - return uploadInfo + return uploadResponse } /* @@ -86,7 +86,7 @@ export async function downloadArtifact( name: string, path?: string, options?: DownloadOptions - ): Promise { + ): Promise { TODO } @@ -95,7 +95,7 @@ Downloads all artifacts associated with a run. Because there are multiple artifa export async function downloadAllArtifacts( path?: string - ): Promise{ + ): Promise{ TODO } diff --git a/packages/artifact/src/download-info.ts b/packages/artifact/src/download-response.ts similarity index 83% rename from packages/artifact/src/download-info.ts rename to packages/artifact/src/download-response.ts index 35b88c24f2..e506967251 100644 --- a/packages/artifact/src/download-info.ts +++ b/packages/artifact/src/download-response.ts @@ -1,4 +1,4 @@ -export interface DownloadInfo { +export interface DownloadResponse { /** * The name of the artifact that was downloaded */ diff --git a/packages/artifact/src/upload-info.ts b/packages/artifact/src/upload-response.ts similarity index 94% rename from packages/artifact/src/upload-info.ts rename to packages/artifact/src/upload-response.ts index abc7ad760f..6887fec840 100644 --- a/packages/artifact/src/upload-info.ts +++ b/packages/artifact/src/upload-response.ts @@ -1,4 +1,4 @@ -export interface UploadInfo { +export interface UploadResponse { /** * The name of the artifact that was uploaded */ From c8d2dd1cf12705299b0727b357d3ec5a88b2e90c Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Fri, 7 Feb 2020 12:05:28 -0500 Subject: [PATCH 35/46] Misc Improvements --- .../artifact/__tests__/upload-specification.test.ts | 4 ++-- packages/artifact/src/upload-http-client.ts | 5 ----- packages/artifact/src/utils.ts | 11 +++++++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/artifact/__tests__/upload-specification.test.ts b/packages/artifact/__tests__/upload-specification.test.ts index 64942c3a66..ae4bfa1759 100644 --- a/packages/artifact/__tests__/upload-specification.test.ts +++ b/packages/artifact/__tests__/upload-specification.test.ts @@ -1,7 +1,7 @@ -import {promises as fs} from 'fs' +import * as io from '../../io/src/io' import * as path from 'path' +import {promises as fs} from 'fs' import {getUploadSpecification} from '../src/upload-specification' -import * as io from '../../io/src/io' const artifactName = 'my-artifact' const root = path.join(__dirname, '_temp', 'upload-specification') diff --git a/packages/artifact/src/upload-http-client.ts b/packages/artifact/src/upload-http-client.ts index dafae76a7a..968155613f 100644 --- a/packages/artifact/src/upload-http-client.ts +++ b/packages/artifact/src/upload-http-client.ts @@ -46,7 +46,6 @@ export async function createArtifactInFileContainer( const body: string = await rawResponse.readBody() if ( - rawResponse.message.statusCode && isSuccessStatusCode(rawResponse.message.statusCode) && body ) { @@ -248,7 +247,6 @@ async function uploadChunk( const response = await uploadChunkRequest() if ( - response.message.statusCode && isSuccessStatusCode(response.message.statusCode) ) { debug( @@ -256,7 +254,6 @@ async function uploadChunk( ) return true } else if ( - response.message.statusCode && isRetryableStatusCode(response.message.statusCode) ) { // eslint-disable-next-line no-console @@ -266,7 +263,6 @@ async function uploadChunk( await new Promise(resolve => setTimeout(resolve, 10000)) const retryResponse = await uploadChunkRequest() if ( - retryResponse.message.statusCode && isSuccessStatusCode(retryResponse.message.statusCode) ) { return true @@ -311,7 +307,6 @@ export async function patchArtifactSize( const body: string = await rawResponse.readBody() if ( - rawResponse.message.statusCode && isSuccessStatusCode(rawResponse.message.statusCode) ) { debug( diff --git a/packages/artifact/src/utils.ts b/packages/artifact/src/utils.ts index e706b4aabf..a35fdb0930 100644 --- a/packages/artifact/src/utils.ts +++ b/packages/artifact/src/utils.ts @@ -26,11 +26,18 @@ export function getApiVersion(): string { return '6.0-preview' } -export function isSuccessStatusCode(statusCode: number): boolean { +export function isSuccessStatusCode(statusCode?: number): boolean { + if (!statusCode) { + return false + } return statusCode >= 200 && statusCode < 300 } -export function isRetryableStatusCode(statusCode: number): boolean { +export function isRetryableStatusCode(statusCode?: number): boolean { + if (!statusCode) { + return false + } + const retryableStatusCodes = [ HttpCodes.BadGateway, HttpCodes.ServiceUnavailable, From 1dd4152b43a9d4c875f63b3d3b25466c22ea4317 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Fri, 7 Feb 2020 12:50:07 -0500 Subject: [PATCH 36/46] Format Fix --- packages/artifact/src/upload-http-client.ts | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/packages/artifact/src/upload-http-client.ts b/packages/artifact/src/upload-http-client.ts index 968155613f..9f0fca4ef6 100644 --- a/packages/artifact/src/upload-http-client.ts +++ b/packages/artifact/src/upload-http-client.ts @@ -45,10 +45,7 @@ export async function createArtifactInFileContainer( const rawResponse = await client.post(artifactUrl, data, requestOptions) const body: string = await rawResponse.readBody() - if ( - isSuccessStatusCode(rawResponse.message.statusCode) && - body - ) { + if (isSuccessStatusCode(rawResponse.message.statusCode) && body) { return JSON.parse(body) } else { // eslint-disable-next-line no-console @@ -246,25 +243,19 @@ async function uploadChunk( } const response = await uploadChunkRequest() - if ( - isSuccessStatusCode(response.message.statusCode) - ) { + if (isSuccessStatusCode(response.message.statusCode)) { debug( `Chunk for ${start}:${end} was successfully uploaded to ${resourceUrl}` ) return true - } else if ( - isRetryableStatusCode(response.message.statusCode) - ) { + } else if (isRetryableStatusCode(response.message.statusCode)) { // eslint-disable-next-line no-console console.log( `Received http ${response.message.statusCode} during chunk upload, will retry at offset ${start} after 10 seconds.` ) await new Promise(resolve => setTimeout(resolve, 10000)) const retryResponse = await uploadChunkRequest() - if ( - isSuccessStatusCode(retryResponse.message.statusCode) - ) { + if (isSuccessStatusCode(retryResponse.message.statusCode)) { return true } else { // eslint-disable-next-line no-console @@ -306,9 +297,7 @@ export async function patchArtifactSize( ) const body: string = await rawResponse.readBody() - if ( - isSuccessStatusCode(rawResponse.message.statusCode) - ) { + if (isSuccessStatusCode(rawResponse.message.statusCode)) { debug( `Artifact ${artifactName} has been successfully uploaded, total size ${size}` ) From 1fa883d80d54181d6e79f335b55459fcd2391fba Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Fri, 7 Feb 2020 14:06:04 -0500 Subject: [PATCH 37/46] Test Improvements --- .../__tests__/upload-specification.test.ts | 51 ++++++++----------- 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/packages/artifact/__tests__/upload-specification.test.ts b/packages/artifact/__tests__/upload-specification.test.ts index ae4bfa1759..08fc8e1c8f 100644 --- a/packages/artifact/__tests__/upload-specification.test.ts +++ b/packages/artifact/__tests__/upload-specification.test.ts @@ -189,28 +189,23 @@ describe('Search', () => { 'good-item1.txt' ) ) - } - if (specification.absoluteFilePath === goodItem2Path) { + } else if (specification.absoluteFilePath === goodItem2Path) { expect(specification.uploadFilePath).toEqual( path.join(artifactName, 'folder-d', 'good-item2.txt') ) - } - if (specification.absoluteFilePath === goodItem3Path) { + } else if (specification.absoluteFilePath === goodItem3Path) { expect(specification.uploadFilePath).toEqual( path.join(artifactName, 'folder-d', 'good-item3.txt') ) - } - if (specification.absoluteFilePath === goodItem4Path) { + } else if (specification.absoluteFilePath === goodItem4Path) { expect(specification.uploadFilePath).toEqual( path.join(artifactName, 'folder-d', 'good-item4.txt') ) - } - if (specification.absoluteFilePath === goodItem5Path) { + } else if (specification.absoluteFilePath === goodItem5Path) { expect(specification.uploadFilePath).toEqual( path.join(artifactName, 'good-item5.txt') ) - } - if (specification.absoluteFilePath === extraFileInFolderCPath) { + } else if (specification.absoluteFilePath === extraFileInFolderCPath) { expect(specification.uploadFilePath).toEqual( path.join( artifactName, @@ -220,11 +215,12 @@ describe('Search', () => { 'extra-file-in-folder-c.txt' ) ) - } - if (specification.absoluteFilePath === amazingFileInFolderHPath) { + } else if (specification.absoluteFilePath === amazingFileInFolderHPath) { expect(specification.uploadFilePath).toEqual( path.join(artifactName, 'folder-h', 'amazing-item.txt') ) + } else { + throw new Error('this should never be reached') } } }) @@ -258,28 +254,23 @@ describe('Search', () => { 'good-item1.txt' ) ) - } - if (specification.absoluteFilePath === goodItem2Path) { + } else if (specification.absoluteFilePath === goodItem2Path) { expect(specification.uploadFilePath).toEqual( path.join(artifactName, 'folder-d', 'good-item2.txt') ) - } - if (specification.absoluteFilePath === goodItem3Path) { + } else if (specification.absoluteFilePath === goodItem3Path) { expect(specification.uploadFilePath).toEqual( path.join(artifactName, 'folder-d', 'good-item3.txt') ) - } - if (specification.absoluteFilePath === goodItem4Path) { + } else if (specification.absoluteFilePath === goodItem4Path) { expect(specification.uploadFilePath).toEqual( path.join(artifactName, 'folder-d', 'good-item4.txt') ) - } - if (specification.absoluteFilePath === goodItem5Path) { + } else if (specification.absoluteFilePath === goodItem5Path) { expect(specification.uploadFilePath).toEqual( path.join(artifactName, 'good-item5.txt') ) - } - if (specification.absoluteFilePath === extraFileInFolderCPath) { + } else if (specification.absoluteFilePath === extraFileInFolderCPath) { expect(specification.uploadFilePath).toEqual( path.join( artifactName, @@ -289,11 +280,12 @@ describe('Search', () => { 'extra-file-in-folder-c.txt' ) ) - } - if (specification.absoluteFilePath === amazingFileInFolderHPath) { + } else if (specification.absoluteFilePath === amazingFileInFolderHPath) { expect(specification.uploadFilePath).toEqual( path.join(artifactName, 'folder-h', 'amazing-item.txt') ) + } else { + throw new Error('this should never be reached') } } }) @@ -328,21 +320,20 @@ describe('Search', () => { 'good-item1.txt' ) ) - } - if (specification.absoluteFilePath === goodItem2Path) { + } else if (specification.absoluteFilePath === goodItem2Path) { expect(specification.uploadFilePath).toEqual( path.join(artifactName, 'folder-d', 'good-item2.txt') ) - } - if (specification.absoluteFilePath === goodItem4Path) { + } else if (specification.absoluteFilePath === goodItem4Path) { expect(specification.uploadFilePath).toEqual( path.join(artifactName, 'folder-d', 'good-item4.txt') ) - } - if (specification.absoluteFilePath === badItem3Path) { + } else if (specification.absoluteFilePath === badItem3Path) { expect(specification.uploadFilePath).toEqual( path.join(artifactName, 'folder-f', 'bad-item3.txt') ) + } else { + throw new Error('this should never be reached') } } }) From 7c6e6acb159ddb6a266e9445ad2741ae64d8f6e2 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Fri, 7 Feb 2020 16:12:20 -0500 Subject: [PATCH 38/46] Updates to continueOnError --- packages/artifact/__tests__/upload.test.ts | 35 +++++++++++++++++++++ packages/artifact/package-lock.json | 14 +++++++-- packages/artifact/package.json | 2 +- packages/artifact/src/config-variables.ts | 2 +- packages/artifact/src/upload-http-client.ts | 17 +++++----- packages/artifact/src/upload-options.ts | 2 +- 6 files changed, 58 insertions(+), 14 deletions(-) diff --git a/packages/artifact/__tests__/upload.test.ts b/packages/artifact/__tests__/upload.test.ts index 7a21ad5af6..f06e74e764 100644 --- a/packages/artifact/__tests__/upload.test.ts +++ b/packages/artifact/__tests__/upload.test.ts @@ -227,6 +227,41 @@ describe('Upload Tests', () => { expect(uploadResult.size).toEqual(expectedPartialSize) }) + it('Upload Artifact - Failed upload with no options', async () => { + const artifactName = 'partial-artifact' + const uploadSpecification: UploadSpecification[] = [ + { + absoluteFilePath: file1Path, + uploadFilePath: `${artifactName}/file1.txt` + }, + { + absoluteFilePath: file2Path, + uploadFilePath: `${artifactName}/file2.txt` + }, + { + absoluteFilePath: file3Path, + uploadFilePath: `${artifactName}/folder1/file3.txt` + }, + { + absoluteFilePath: file4Path, + uploadFilePath: `this-file-upload-will-fail` + }, + { + absoluteFilePath: file5Path, + uploadFilePath: `${artifactName}/folder1/folder2/folder3/file5.txt` + } + ] + + const expectedPartialSize = file1Size + file2Size + file3Size + file5Size + const uploadUrl = `${getRuntimeUrl()}_apis/resources/Containers/13` + const uploadResult = await uploadHttpClient.uploadArtifactToFileContainer( + uploadUrl, + uploadSpecification + ) + expect(uploadResult.failedItems.length).toEqual(1) + expect(uploadResult.size).toEqual(expectedPartialSize) + }) + /** * Artifact Association Tests */ diff --git a/packages/artifact/package-lock.json b/packages/artifact/package-lock.json index 8e26600de7..d8cd2feb44 100644 --- a/packages/artifact/package-lock.json +++ b/packages/artifact/package-lock.json @@ -10,9 +10,17 @@ "integrity": "sha512-xD+CQd9p4lU7ZfRqmUcbJpqR+Ss51rJRVeXMyOLrZQImN9/8Sy/BEUBnHO/UKD3z03R686PVTLfEPmkropGuLw==" }, "@actions/http-client": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-1.0.1.tgz", - "integrity": "sha512-vy5DhqTJ1gtEkpRrD/6BHhUlkeyccrOX0BT9KmtO5TWxe5KSSwVHFE+J15Z0dG+tJwZJ/nHC4slUIyqpkahoMg==" + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-1.0.6.tgz", + "integrity": "sha512-LGmio4w98UyGX33b/W6V6Nx/sQHRXZ859YlMkn36wPsXPB82u8xTVlA/Dq2DXrm6lEq9RVmisRJa1c+HETAIJA==", + "requires": { + "tunnel": "0.0.6" + } + }, + "tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" } } } diff --git a/packages/artifact/package.json b/packages/artifact/package.json index c99b800b1a..0f38fa9b67 100644 --- a/packages/artifact/package.json +++ b/packages/artifact/package.json @@ -37,6 +37,6 @@ }, "dependencies": { "@actions/core": "^1.2.1", - "@actions/http-client": "^1.0.1" + "@actions/http-client": "^1.0.6" } } diff --git a/packages/artifact/src/config-variables.ts b/packages/artifact/src/config-variables.ts index 45022ed5d4..aef9d34c0e 100644 --- a/packages/artifact/src/config-variables.ts +++ b/packages/artifact/src/config-variables.ts @@ -3,7 +3,7 @@ export function getUploadFileConcurrency(): number { } export function getUploadChunkConcurrency(): number { - return 3 + return 1 } export function getUploadChunkSize(): number { diff --git a/packages/artifact/src/upload-http-client.ts b/packages/artifact/src/upload-http-client.ts index 9f0fca4ef6..99282ab255 100644 --- a/packages/artifact/src/upload-http-client.ts +++ b/packages/artifact/src/upload-http-client.ts @@ -76,9 +76,13 @@ export async function uploadArtifactToFileContainer( ) const parameters: UploadFileParameters[] = [] + + // by default, file uploads will continue if there is an error unless specified differently in the options let continueOnError = true if (options) { - continueOnError = options.continueOnError + if (options.continueOnError === false) { + continueOnError = false + } } // Prepare the necessary parameters to upload all the files @@ -181,17 +185,14 @@ async function uploadFileAsync( if (!result) { /** - * Chunk failed to upload, report as failed but continue if desired. It is possible that part of a chunk was + * Chunk failed to upload, report as failed and do not continue uploading any more chunks for the file. It is possible that part of a chunk was * successfully uploaded so the server may report a different size for what was uploaded **/ isUploadSuccessful = false failedChunkSizes += chunkSize - if (!parameters.continueOnError) { - // Any currently uploading chunks will be able to finish, however pending chunks will not upload - // eslint-disable-next-line no-console - console.log(`Aborting upload for ${parameters.file} due to failure`) - abortFileUpload = true - } + // eslint-disable-next-line no-console + console.log(`Aborting upload for ${parameters.file} due to failure`) + abortFileUpload = true } } }) diff --git a/packages/artifact/src/upload-options.ts b/packages/artifact/src/upload-options.ts index 0792010317..63d4febe8f 100644 --- a/packages/artifact/src/upload-options.ts +++ b/packages/artifact/src/upload-options.ts @@ -14,5 +14,5 @@ export interface UploadOptions { * files with the exception of the problematic files(s)/chunks(s) that failed to upload * */ - continueOnError: boolean + continueOnError?: boolean } From 6b6a5e01c9aeccf80dffde1829051740d00f234e Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Fri, 7 Feb 2020 16:32:28 -0500 Subject: [PATCH 39/46] Extra test --- packages/artifact/__tests__/upload.test.ts | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/packages/artifact/__tests__/upload.test.ts b/packages/artifact/__tests__/upload.test.ts index f06e74e764..49a62d84aa 100644 --- a/packages/artifact/__tests__/upload.test.ts +++ b/packages/artifact/__tests__/upload.test.ts @@ -262,6 +262,42 @@ describe('Upload Tests', () => { expect(uploadResult.size).toEqual(expectedPartialSize) }) + it('Upload Artifact - Failed upload with empty options', async () => { + const artifactName = 'partial-artifact' + const uploadSpecification: UploadSpecification[] = [ + { + absoluteFilePath: file1Path, + uploadFilePath: `${artifactName}/file1.txt` + }, + { + absoluteFilePath: file2Path, + uploadFilePath: `${artifactName}/file2.txt` + }, + { + absoluteFilePath: file3Path, + uploadFilePath: `${artifactName}/folder1/file3.txt` + }, + { + absoluteFilePath: file4Path, + uploadFilePath: `this-file-upload-will-fail` + }, + { + absoluteFilePath: file5Path, + uploadFilePath: `${artifactName}/folder1/folder2/folder3/file5.txt` + } + ] + + const expectedPartialSize = file1Size + file2Size + file3Size + file5Size + const uploadUrl = `${getRuntimeUrl()}_apis/resources/Containers/13` + const uploadResult = await uploadHttpClient.uploadArtifactToFileContainer( + uploadUrl, + uploadSpecification, + {} + ) + expect(uploadResult.failedItems.length).toEqual(1) + expect(uploadResult.size).toEqual(expectedPartialSize) + }) + /** * Artifact Association Tests */ From 4ac6ef63d8a99bfac7d254076abe1954dc49f489 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Fri, 7 Feb 2020 16:58:44 -0500 Subject: [PATCH 40/46] Normalize and Resolve all paths --- packages/artifact/src/artifact.ts | 4 ++-- packages/artifact/src/upload-specification.ts | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/artifact/src/artifact.ts b/packages/artifact/src/artifact.ts index 5a40016f23..51f1514102 100644 --- a/packages/artifact/src/artifact.ts +++ b/packages/artifact/src/artifact.ts @@ -16,8 +16,8 @@ import {checkArtifactName} from './utils' * Uploads an artifact * * @param name the name of the artifact, required - * @param files a list of absolute paths that denote what files should be uploaded - * @param rootDirectory an absolute file path that denotes the root parent directory of the files being uploaded + * @param files a list of absolute or relative paths that denote what files should be uploaded + * @param rootDirectory an absolute or relative file path that denotes the root parent directory of the files being uploaded * @param options extra options for customizing the upload behavior * @returns single UploadInfo object */ diff --git a/packages/artifact/src/upload-specification.ts b/packages/artifact/src/upload-specification.ts index 40456cdc03..77831817ad 100644 --- a/packages/artifact/src/upload-specification.ts +++ b/packages/artifact/src/upload-specification.ts @@ -1,6 +1,6 @@ import * as fs from 'fs' import {debug} from '@actions/core' -import {join, normalize} from 'path' +import {join, normalize, resolve} from 'path' import {checkArtifactName} from './utils' export interface UploadSpecification { @@ -31,7 +31,10 @@ export function getUploadSpecification( `Provided rootDirectory ${rootDirectory} is not a valid directory` ) } + // Normalize and resolve, this allows for either absolute or relative paths to be used rootDirectory = normalize(rootDirectory) + rootDirectory = resolve(rootDirectory) + /* Example to demonstrate behavior @@ -57,7 +60,9 @@ export function getUploadSpecification( } if (!fs.lstatSync(file).isDirectory()) { + // Normalize and resolve, this allows for either absolute or relative paths to be used file = normalize(file) + file = resolve(file) if (!file.startsWith(rootDirectory)) { throw new Error( `The rootDirectory: ${rootDirectory} is not a parent directory of the file: ${file}` From d01f0f303b13d4baa27aa34446a26c983f69347e Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Fri, 7 Feb 2020 17:14:31 -0500 Subject: [PATCH 41/46] Change console.log to warning --- packages/artifact/src/upload-http-client.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/artifact/src/upload-http-client.ts b/packages/artifact/src/upload-http-client.ts index 99282ab255..df7a558591 100644 --- a/packages/artifact/src/upload-http-client.ts +++ b/packages/artifact/src/upload-http-client.ts @@ -1,4 +1,4 @@ -import {debug} from '@actions/core' +import {debug, warning} from '@actions/core' import {HttpClientResponse, HttpClient} from '@actions/http-client/index' import {IHttpClientResponse} from '@actions/http-client/interfaces' import { @@ -190,8 +190,7 @@ async function uploadFileAsync( **/ isUploadSuccessful = false failedChunkSizes += chunkSize - // eslint-disable-next-line no-console - console.log(`Aborting upload for ${parameters.file} due to failure`) + warning(`Aborting upload for ${parameters.file} due to failure`) abortFileUpload = true } } From 791775489a55862ef92e708c8a209c580dddca4e Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Mon, 10 Feb 2020 10:49:00 -0500 Subject: [PATCH 42/46] Update upload-response.ts --- packages/artifact/src/upload-response.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/artifact/src/upload-response.ts b/packages/artifact/src/upload-response.ts index 6887fec840..f40379edad 100644 --- a/packages/artifact/src/upload-response.ts +++ b/packages/artifact/src/upload-response.ts @@ -5,7 +5,7 @@ export interface UploadResponse { artifactName: string /** - * A list of all items found using the provided path that are intended to be uploaded as part of the artifact + * A list of all items that are meant to be uploaded as part of the artifact */ artifactItems: string[] From 21ce5cbba3cff2c304c2d8941941c02a9ee43a4a Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Mon, 10 Feb 2020 14:38:10 -0500 Subject: [PATCH 43/46] PR feedback --- .../__tests__/upload-specification.test.ts | 46 +++++++++++-------- packages/artifact/src/artifact.ts | 3 +- packages/artifact/src/upload-http-client.ts | 11 ++--- 3 files changed, 31 insertions(+), 29 deletions(-) diff --git a/packages/artifact/__tests__/upload-specification.test.ts b/packages/artifact/__tests__/upload-specification.test.ts index 08fc8e1c8f..d4b1836b7d 100644 --- a/packages/artifact/__tests__/upload-specification.test.ts +++ b/packages/artifact/__tests__/upload-specification.test.ts @@ -170,13 +170,13 @@ describe('Search', () => { expect(specifications.length).toEqual(7) const absolutePaths = specifications.map(item => item.absoluteFilePath) - expect(absolutePaths.includes(goodItem1Path)).toEqual(true) - expect(absolutePaths.includes(goodItem2Path)).toEqual(true) - expect(absolutePaths.includes(goodItem3Path)).toEqual(true) - expect(absolutePaths.includes(goodItem4Path)).toEqual(true) - expect(absolutePaths.includes(goodItem5Path)).toEqual(true) - expect(absolutePaths.includes(extraFileInFolderCPath)).toEqual(true) - expect(absolutePaths.includes(amazingFileInFolderHPath)).toEqual(true) + expect(absolutePaths).toContain(goodItem1Path) + expect(absolutePaths).toContain(goodItem2Path) + expect(absolutePaths).toContain(goodItem3Path) + expect(absolutePaths).toContain(goodItem4Path) + expect(absolutePaths).toContain(goodItem5Path) + expect(absolutePaths).toContain(extraFileInFolderCPath) + expect(absolutePaths).toContain(amazingFileInFolderHPath) for (const specification of specifications) { if (specification.absoluteFilePath === goodItem1Path) { @@ -220,7 +220,9 @@ describe('Search', () => { path.join(artifactName, 'folder-h', 'amazing-item.txt') ) } else { - throw new Error('this should never be reached') + throw new Error( + 'Invalid specification found. This should never be reached' + ) } } }) @@ -235,13 +237,13 @@ describe('Search', () => { expect(specifications.length).toEqual(7) const absolutePaths = specifications.map(item => item.absoluteFilePath) - expect(absolutePaths.includes(goodItem1Path)).toEqual(true) - expect(absolutePaths.includes(goodItem2Path)).toEqual(true) - expect(absolutePaths.includes(goodItem3Path)).toEqual(true) - expect(absolutePaths.includes(goodItem4Path)).toEqual(true) - expect(absolutePaths.includes(goodItem5Path)).toEqual(true) - expect(absolutePaths.includes(extraFileInFolderCPath)).toEqual(true) - expect(absolutePaths.includes(amazingFileInFolderHPath)).toEqual(true) + expect(absolutePaths).toContain(goodItem1Path) + expect(absolutePaths).toContain(goodItem2Path) + expect(absolutePaths).toContain(goodItem3Path) + expect(absolutePaths).toContain(goodItem4Path) + expect(absolutePaths).toContain(goodItem5Path) + expect(absolutePaths).toContain(extraFileInFolderCPath) + expect(absolutePaths).toContain(amazingFileInFolderHPath) for (const specification of specifications) { if (specification.absoluteFilePath === goodItem1Path) { @@ -285,7 +287,9 @@ describe('Search', () => { path.join(artifactName, 'folder-h', 'amazing-item.txt') ) } else { - throw new Error('this should never be reached') + throw new Error( + 'Invalid specification found. This should never be reached' + ) } } }) @@ -305,9 +309,9 @@ describe('Search', () => { ) expect(specifications.length).toEqual(3) const absolutePaths = specifications.map(item => item.absoluteFilePath) - expect(absolutePaths.includes(goodItem1Path)).toEqual(true) - expect(absolutePaths.includes(goodItem4Path)).toEqual(true) - expect(absolutePaths.includes(badItem3Path)).toEqual(true) + expect(absolutePaths).toContain(goodItem1Path) + expect(absolutePaths).toContain(goodItem4Path) + expect(absolutePaths).toContain(badItem3Path) for (const specification of specifications) { if (specification.absoluteFilePath === goodItem1Path) { @@ -333,7 +337,9 @@ describe('Search', () => { path.join(artifactName, 'folder-f', 'bad-item3.txt') ) } else { - throw new Error('this should never be reached') + throw new Error( + 'Invalid specification found. This should never be reached' + ) } } }) diff --git a/packages/artifact/src/artifact.ts b/packages/artifact/src/artifact.ts index 51f1514102..0e679d0fc2 100644 --- a/packages/artifact/src/artifact.ts +++ b/packages/artifact/src/artifact.ts @@ -65,8 +65,7 @@ export async function uploadArtifact( //Update the size of the artifact to indicate we are done uploading await patchArtifactSize(uploadResult.size, name) - // eslint-disable-next-line no-console - console.log( + core.info( `Finished uploading artifact ${name}. Reported size is ${uploadResult.size} bytes. There were ${uploadResult.failedItems.length} items that failed to upload` ) diff --git a/packages/artifact/src/upload-http-client.ts b/packages/artifact/src/upload-http-client.ts index df7a558591..f9f96a59d5 100644 --- a/packages/artifact/src/upload-http-client.ts +++ b/packages/artifact/src/upload-http-client.ts @@ -1,4 +1,4 @@ -import {debug, warning} from '@actions/core' +import {debug, warning, info} from '@actions/core' import {HttpClientResponse, HttpClient} from '@actions/http-client/index' import {IHttpClientResponse} from '@actions/http-client/interfaces' import { @@ -129,8 +129,7 @@ export async function uploadArtifactToFileContainer( }) ) - // eslint-disable-next-line no-console - console.log(`Total size of all the files uploaded is ${fileSizes} bytes`) + info(`Total size of all the files uploaded is ${fileSizes} bytes`) return { size: fileSizes, failedItems: failedItemsToReport @@ -249,8 +248,7 @@ async function uploadChunk( ) return true } else if (isRetryableStatusCode(response.message.statusCode)) { - // eslint-disable-next-line no-console - console.log( + info( `Received http ${response.message.statusCode} during chunk upload, will retry at offset ${start} after 10 seconds.` ) await new Promise(resolve => setTimeout(resolve, 10000)) @@ -258,8 +256,7 @@ async function uploadChunk( if (isSuccessStatusCode(retryResponse.message.statusCode)) { return true } else { - // eslint-disable-next-line no-console - console.log(`Unable to upload chunk even after retrying`) + info(`Unable to upload chunk even after retrying`) // eslint-disable-next-line no-console console.log(response) return false From fcec6a7a0816525dfa42c67a85b86d851679a7ee Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Mon, 10 Feb 2020 14:49:16 -0500 Subject: [PATCH 44/46] Update upload-http-client.ts --- packages/artifact/src/upload-http-client.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/artifact/src/upload-http-client.ts b/packages/artifact/src/upload-http-client.ts index f9f96a59d5..9828b3bfde 100644 --- a/packages/artifact/src/upload-http-client.ts +++ b/packages/artifact/src/upload-http-client.ts @@ -220,8 +220,7 @@ async function uploadChunk( end: number, totalSize: number ): Promise { - // eslint-disable-next-line no-console - console.log( + info( `Uploading chunk of size ${end - start + 1} bytes at offset ${start} with content range: ${getContentRange( From 52885336a487e6a8054410f433c288f72a16881c Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Mon, 10 Feb 2020 16:21:43 -0500 Subject: [PATCH 45/46] Silence output when running tests and rename files with internal --- .../__tests__/upload-specification.test.ts | 9 ++++++++- packages/artifact/__tests__/upload.test.ts | 17 ++++++++++++----- packages/artifact/__tests__/util.test.ts | 16 +++++++++++++--- ...ariables.ts => internal-config-variables.ts} | 0 ...ariables.ts => internal-config-variables.ts} | 0 .../src/{contracts.ts => internal-contracts.ts} | 0 ...-options.ts => internal-download-options.ts} | 0 ...esponse.ts => internal-download-response.ts} | 0 ...client.ts => internal-upload-http-client.ts} | 10 +++++----- ...ad-options.ts => internal-upload-options.ts} | 0 ...-response.ts => internal-upload-response.ts} | 0 ...tion.ts => internal-upload-specification.ts} | 2 +- .../src/{utils.ts => internal-utils.ts} | 2 +- 13 files changed, 40 insertions(+), 16 deletions(-) rename packages/artifact/src/__mocks__/{config-variables.ts => internal-config-variables.ts} (100%) rename packages/artifact/src/{config-variables.ts => internal-config-variables.ts} (100%) rename packages/artifact/src/{contracts.ts => internal-contracts.ts} (100%) rename packages/artifact/src/{download-options.ts => internal-download-options.ts} (100%) rename packages/artifact/src/{download-response.ts => internal-download-response.ts} (100%) rename packages/artifact/src/{upload-http-client.ts => internal-upload-http-client.ts} (98%) rename packages/artifact/src/{upload-options.ts => internal-upload-options.ts} (100%) rename packages/artifact/src/{upload-response.ts => internal-upload-response.ts} (100%) rename packages/artifact/src/{upload-specification.ts => internal-upload-specification.ts} (98%) rename packages/artifact/src/{utils.ts => internal-utils.ts} (98%) diff --git a/packages/artifact/__tests__/upload-specification.test.ts b/packages/artifact/__tests__/upload-specification.test.ts index d4b1836b7d..cc70853894 100644 --- a/packages/artifact/__tests__/upload-specification.test.ts +++ b/packages/artifact/__tests__/upload-specification.test.ts @@ -1,7 +1,8 @@ import * as io from '../../io/src/io' import * as path from 'path' import {promises as fs} from 'fs' -import {getUploadSpecification} from '../src/upload-specification' +import * as core from '@actions/core' +import {getUploadSpecification} from '../src/internal-upload-specification' const artifactName = 'my-artifact' const root = path.join(__dirname, '_temp', 'upload-specification') @@ -48,6 +49,12 @@ const artifactFilesToUpload = [ describe('Search', () => { beforeAll(async () => { + // mock all output so that there is less noise when running tests + console.log = jest.fn() + jest.spyOn(core, 'debug').mockImplementation(() => {}) + jest.spyOn(core, 'info').mockImplementation(() => {}) + jest.spyOn(core, 'warning').mockImplementation(() => {}) + // clear temp directory await io.rmRF(root) await fs.mkdir(path.join(root, 'folder-a', 'folder-b', 'folder-c'), { diff --git a/packages/artifact/__tests__/upload.test.ts b/packages/artifact/__tests__/upload.test.ts index 49a62d84aa..e17f9ee18d 100644 --- a/packages/artifact/__tests__/upload.test.ts +++ b/packages/artifact/__tests__/upload.test.ts @@ -2,15 +2,16 @@ import * as http from 'http' import * as io from '../../io/src/io' import * as net from 'net' import * as path from 'path' -import * as uploadHttpClient from '../src/upload-http-client' +import * as uploadHttpClient from '../src/internal-upload-http-client' +import * as core from '@actions/core' import {promises as fs} from 'fs' -import {getRuntimeUrl} from '../src/config-variables' +import {getRuntimeUrl} from '../src/internal-config-variables' import {HttpClient, HttpClientResponse} from '@actions/http-client' import { ArtifactResponse, PatchArtifactSizeSuccessResponse -} from '../src/contracts' -import {UploadSpecification} from '../src/upload-specification' +} from '../src/internal-contracts' +import {UploadSpecification} from '../src/internal-upload-specification' const root = path.join(__dirname, '_temp', 'artifact-upload') const file1Path = path.join(root, 'file1.txt') @@ -25,11 +26,17 @@ let file3Size = 0 let file4Size = 0 let file5Size = 0 -jest.mock('../src/config-variables') +jest.mock('../src/internal-config-variables') jest.mock('@actions/http-client') describe('Upload Tests', () => { beforeAll(async () => { + // mock all output so that there is less noise when running tests + console.log = jest.fn() + jest.spyOn(core, 'debug').mockImplementation(() => {}) + jest.spyOn(core, 'info').mockImplementation(() => {}) + jest.spyOn(core, 'warning').mockImplementation(() => {}) + // setup mocking for calls that got through the HttpClient setupHttpClientMock() diff --git a/packages/artifact/__tests__/util.test.ts b/packages/artifact/__tests__/util.test.ts index 68d2577d5e..dc8e5cdd29 100644 --- a/packages/artifact/__tests__/util.test.ts +++ b/packages/artifact/__tests__/util.test.ts @@ -1,10 +1,20 @@ -import * as utils from '../src/utils' +import * as utils from '../src/internal-utils' +import * as core from '@actions/core' import {HttpCodes} from '@actions/http-client' -import {getRuntimeUrl, getWorkFlowRunId} from '../src/config-variables' +import {getRuntimeUrl, getWorkFlowRunId} from '../src/internal-config-variables' -jest.mock('../src/config-variables') +jest.mock('../src/internal-config-variables') describe('Utils', () => { + + beforeAll(() => { + // mock all output so that there is less noise when running tests + console.log = jest.fn() + jest.spyOn(core, 'debug').mockImplementation(() => {}) + jest.spyOn(core, 'info').mockImplementation(() => {}) + jest.spyOn(core, 'warning').mockImplementation(() => {}) + }) + it('Check Artifact Name for any invalid characters', () => { const invalidNames = [ 'my\\artifact', diff --git a/packages/artifact/src/__mocks__/config-variables.ts b/packages/artifact/src/__mocks__/internal-config-variables.ts similarity index 100% rename from packages/artifact/src/__mocks__/config-variables.ts rename to packages/artifact/src/__mocks__/internal-config-variables.ts diff --git a/packages/artifact/src/config-variables.ts b/packages/artifact/src/internal-config-variables.ts similarity index 100% rename from packages/artifact/src/config-variables.ts rename to packages/artifact/src/internal-config-variables.ts diff --git a/packages/artifact/src/contracts.ts b/packages/artifact/src/internal-contracts.ts similarity index 100% rename from packages/artifact/src/contracts.ts rename to packages/artifact/src/internal-contracts.ts diff --git a/packages/artifact/src/download-options.ts b/packages/artifact/src/internal-download-options.ts similarity index 100% rename from packages/artifact/src/download-options.ts rename to packages/artifact/src/internal-download-options.ts diff --git a/packages/artifact/src/download-response.ts b/packages/artifact/src/internal-download-response.ts similarity index 100% rename from packages/artifact/src/download-response.ts rename to packages/artifact/src/internal-download-response.ts diff --git a/packages/artifact/src/upload-http-client.ts b/packages/artifact/src/internal-upload-http-client.ts similarity index 98% rename from packages/artifact/src/upload-http-client.ts rename to packages/artifact/src/internal-upload-http-client.ts index f9f96a59d5..1072b76139 100644 --- a/packages/artifact/src/upload-http-client.ts +++ b/packages/artifact/src/internal-upload-http-client.ts @@ -6,10 +6,10 @@ import { CreateArtifactParameters, PatchArtifactSize, UploadResults -} from './contracts' +} from './internal-contracts' import * as fs from 'fs' -import {UploadSpecification} from './upload-specification' -import {UploadOptions} from './upload-options' +import {UploadSpecification} from './internal-upload-specification' +import {UploadOptions} from './internal-upload-options' import {URL} from 'url' import { createHttpClient, @@ -18,12 +18,12 @@ import { getRequestOptions, isRetryableStatusCode, isSuccessStatusCode -} from './utils' +} from './internal-utils' import { getUploadChunkConcurrency, getUploadChunkSize, getUploadFileConcurrency -} from './config-variables' +} from './internal-config-variables' /** * Creates a file container for the new artifact in the remote blob storage/file service diff --git a/packages/artifact/src/upload-options.ts b/packages/artifact/src/internal-upload-options.ts similarity index 100% rename from packages/artifact/src/upload-options.ts rename to packages/artifact/src/internal-upload-options.ts diff --git a/packages/artifact/src/upload-response.ts b/packages/artifact/src/internal-upload-response.ts similarity index 100% rename from packages/artifact/src/upload-response.ts rename to packages/artifact/src/internal-upload-response.ts diff --git a/packages/artifact/src/upload-specification.ts b/packages/artifact/src/internal-upload-specification.ts similarity index 98% rename from packages/artifact/src/upload-specification.ts rename to packages/artifact/src/internal-upload-specification.ts index 77831817ad..0df866662b 100644 --- a/packages/artifact/src/upload-specification.ts +++ b/packages/artifact/src/internal-upload-specification.ts @@ -1,7 +1,7 @@ import * as fs from 'fs' import {debug} from '@actions/core' import {join, normalize, resolve} from 'path' -import {checkArtifactName} from './utils' +import {checkArtifactName} from './internal-utils' export interface UploadSpecification { absoluteFilePath: string diff --git a/packages/artifact/src/utils.ts b/packages/artifact/src/internal-utils.ts similarity index 98% rename from packages/artifact/src/utils.ts rename to packages/artifact/src/internal-utils.ts index a35fdb0930..afc2434270 100644 --- a/packages/artifact/src/utils.ts +++ b/packages/artifact/src/internal-utils.ts @@ -6,7 +6,7 @@ import { getRuntimeToken, getRuntimeUrl, getWorkFlowRunId -} from './config-variables' +} from './internal-config-variables' /** * Parses a env variable that is a number From 8a1220b0034b4199ec7a3ff3d8076eac92fbbce0 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Mon, 10 Feb 2020 16:56:05 -0500 Subject: [PATCH 46/46] Use Factory to create ArtifactClient --- .../__tests__/upload-specification.test.ts | 2 +- packages/artifact/__tests__/upload.test.ts | 2 +- packages/artifact/__tests__/util.test.ts | 3 +- packages/artifact/src/artifact-client.ts | 9 ++ packages/artifact/src/artifact.ts | 101 -------------- .../artifact/src/internal-artifact-client.ts | 124 ++++++++++++++++++ 6 files changed, 136 insertions(+), 105 deletions(-) create mode 100644 packages/artifact/src/artifact-client.ts delete mode 100644 packages/artifact/src/artifact.ts create mode 100644 packages/artifact/src/internal-artifact-client.ts diff --git a/packages/artifact/__tests__/upload-specification.test.ts b/packages/artifact/__tests__/upload-specification.test.ts index cc70853894..ba1cd9e435 100644 --- a/packages/artifact/__tests__/upload-specification.test.ts +++ b/packages/artifact/__tests__/upload-specification.test.ts @@ -50,7 +50,7 @@ const artifactFilesToUpload = [ describe('Search', () => { beforeAll(async () => { // mock all output so that there is less noise when running tests - console.log = jest.fn() + jest.spyOn(console, 'log').mockImplementation(() => {}) jest.spyOn(core, 'debug').mockImplementation(() => {}) jest.spyOn(core, 'info').mockImplementation(() => {}) jest.spyOn(core, 'warning').mockImplementation(() => {}) diff --git a/packages/artifact/__tests__/upload.test.ts b/packages/artifact/__tests__/upload.test.ts index e17f9ee18d..08a758ca4e 100644 --- a/packages/artifact/__tests__/upload.test.ts +++ b/packages/artifact/__tests__/upload.test.ts @@ -32,7 +32,7 @@ jest.mock('@actions/http-client') describe('Upload Tests', () => { beforeAll(async () => { // mock all output so that there is less noise when running tests - console.log = jest.fn() + jest.spyOn(console, 'log').mockImplementation(() => {}) jest.spyOn(core, 'debug').mockImplementation(() => {}) jest.spyOn(core, 'info').mockImplementation(() => {}) jest.spyOn(core, 'warning').mockImplementation(() => {}) diff --git a/packages/artifact/__tests__/util.test.ts b/packages/artifact/__tests__/util.test.ts index dc8e5cdd29..e71652b3c2 100644 --- a/packages/artifact/__tests__/util.test.ts +++ b/packages/artifact/__tests__/util.test.ts @@ -6,10 +6,9 @@ import {getRuntimeUrl, getWorkFlowRunId} from '../src/internal-config-variables' jest.mock('../src/internal-config-variables') describe('Utils', () => { - beforeAll(() => { // mock all output so that there is less noise when running tests - console.log = jest.fn() + jest.spyOn(console, 'log').mockImplementation(() => {}) jest.spyOn(core, 'debug').mockImplementation(() => {}) jest.spyOn(core, 'info').mockImplementation(() => {}) jest.spyOn(core, 'warning').mockImplementation(() => {}) diff --git a/packages/artifact/src/artifact-client.ts b/packages/artifact/src/artifact-client.ts new file mode 100644 index 0000000000..a7ddc69d9c --- /dev/null +++ b/packages/artifact/src/artifact-client.ts @@ -0,0 +1,9 @@ +import {ArtifactClient, DefaultArtifactClient} from './internal-artifact-client' +export {ArtifactClient} + +/** + * Constructs an ArtifactClient + */ +export function create(): ArtifactClient { + return DefaultArtifactClient.create() +} diff --git a/packages/artifact/src/artifact.ts b/packages/artifact/src/artifact.ts deleted file mode 100644 index 0e679d0fc2..0000000000 --- a/packages/artifact/src/artifact.ts +++ /dev/null @@ -1,101 +0,0 @@ -import * as core from '@actions/core' -import { - UploadSpecification, - getUploadSpecification -} from './upload-specification' -import { - createArtifactInFileContainer, - uploadArtifactToFileContainer, - patchArtifactSize -} from './upload-http-client' -import {UploadResponse} from './upload-response' -import {UploadOptions} from './upload-options' -import {checkArtifactName} from './utils' - -/** - * Uploads an artifact - * - * @param name the name of the artifact, required - * @param files a list of absolute or relative paths that denote what files should be uploaded - * @param rootDirectory an absolute or relative file path that denotes the root parent directory of the files being uploaded - * @param options extra options for customizing the upload behavior - * @returns single UploadInfo object - */ -export async function uploadArtifact( - name: string, - files: string[], - rootDirectory: string, - options?: UploadOptions -): Promise { - checkArtifactName(name) - - // Get specification for the files being uploaded - const uploadSpecification: UploadSpecification[] = getUploadSpecification( - name, - rootDirectory, - files - ) - const uploadResponse: UploadResponse = { - artifactName: name, - artifactItems: [], - size: 0, - failedItems: [] - } - - if (uploadSpecification.length === 0) { - core.warning(`No files found that can be uploaded`) - } else { - // Create an entry for the artifact in the file container - const response = await createArtifactInFileContainer(name) - if (!response.fileContainerResourceUrl) { - core.debug(response.toString()) - throw new Error( - 'No URL provided by the Artifact Service to upload an artifact to' - ) - } - core.debug(`Upload Resource URL: ${response.fileContainerResourceUrl}`) - - // Upload each of the files that were found concurrently - const uploadResult = await uploadArtifactToFileContainer( - response.fileContainerResourceUrl, - uploadSpecification, - options - ) - - //Update the size of the artifact to indicate we are done uploading - await patchArtifactSize(uploadResult.size, name) - - core.info( - `Finished uploading artifact ${name}. Reported size is ${uploadResult.size} bytes. There were ${uploadResult.failedItems.length} items that failed to upload` - ) - - uploadResponse.artifactItems = uploadSpecification.map( - item => item.absoluteFilePath - ) - uploadResponse.size = uploadResult.size - uploadResponse.failedItems = uploadResult.failedItems - } - return uploadResponse -} - -/* -Downloads a single artifact associated with a run - -export async function downloadArtifact( - name: string, - path?: string, - options?: DownloadOptions - ): Promise { - - TODO -} - -Downloads all artifacts associated with a run. Because there are multiple artifacts being downloaded, a folder will be created for each one in the specified or default directory - -export async function downloadAllArtifacts( - path?: string - ): Promise{ - - TODO -} -*/ diff --git a/packages/artifact/src/internal-artifact-client.ts b/packages/artifact/src/internal-artifact-client.ts new file mode 100644 index 0000000000..bf431861e8 --- /dev/null +++ b/packages/artifact/src/internal-artifact-client.ts @@ -0,0 +1,124 @@ +import * as core from '@actions/core' +import { + UploadSpecification, + getUploadSpecification +} from './internal-upload-specification' +import { + createArtifactInFileContainer, + uploadArtifactToFileContainer, + patchArtifactSize +} from './internal-upload-http-client' +import {UploadResponse} from './internal-upload-response' +import {UploadOptions} from './internal-upload-options' +import {checkArtifactName} from './internal-utils' + +export {UploadResponse, UploadOptions} + +export interface ArtifactClient { + /** + * Uploads an artifact + * + * @param name the name of the artifact, required + * @param files a list of absolute or relative paths that denote what files should be uploaded + * @param rootDirectory an absolute or relative file path that denotes the root parent directory of the files being uploaded + * @param options extra options for customizing the upload behavior + * @returns single UploadInfo object + */ + uploadArtifact( + name: string, + files: string[], + rootDirectory: string, + options?: UploadOptions + ): Promise +} + +export class DefaultArtifactClient implements ArtifactClient { + /** + * Constructs a DefaultArtifactClient + */ + static create(): DefaultArtifactClient { + return new DefaultArtifactClient() + } + + /** + * Uploads an artifact + */ + async uploadArtifact( + name: string, + files: string[], + rootDirectory: string, + options?: UploadOptions | undefined + ): Promise { + checkArtifactName(name) + + // Get specification for the files being uploaded + const uploadSpecification: UploadSpecification[] = getUploadSpecification( + name, + rootDirectory, + files + ) + const uploadResponse: UploadResponse = { + artifactName: name, + artifactItems: [], + size: 0, + failedItems: [] + } + + if (uploadSpecification.length === 0) { + core.warning(`No files found that can be uploaded`) + } else { + // Create an entry for the artifact in the file container + const response = await createArtifactInFileContainer(name) + if (!response.fileContainerResourceUrl) { + core.debug(response.toString()) + throw new Error( + 'No URL provided by the Artifact Service to upload an artifact to' + ) + } + core.debug(`Upload Resource URL: ${response.fileContainerResourceUrl}`) + + // Upload each of the files that were found concurrently + const uploadResult = await uploadArtifactToFileContainer( + response.fileContainerResourceUrl, + uploadSpecification, + options + ) + + //Update the size of the artifact to indicate we are done uploading + await patchArtifactSize(uploadResult.size, name) + + core.info( + `Finished uploading artifact ${name}. Reported size is ${uploadResult.size} bytes. There were ${uploadResult.failedItems.length} items that failed to upload` + ) + + uploadResponse.artifactItems = uploadSpecification.map( + item => item.absoluteFilePath + ) + uploadResponse.size = uploadResult.size + uploadResponse.failedItems = uploadResult.failedItems + } + return uploadResponse + } + + /* + Downloads a single artifact associated with a run + + export async function downloadArtifact( + name: string, + path?: string, + options?: DownloadOptions + ): Promise { + + TODO + } + + Downloads all artifacts associated with a run. Because there are multiple artifacts being downloaded, a folder will be created for each one in the specified or default directory + + export async function downloadAllArtifacts( + path?: string + ): Promise{ + + TODO + } + */ +}