diff --git a/dist/index.js b/dist/index.js index 42fdfff..c82d2fa 100644 --- a/dist/index.js +++ b/dist/index.js @@ -684,10 +684,7 @@ function getCacheVersion(paths, compressionMethod, enableCrossOsArchive = false) } // Add salt to cache version to support breaking changes in cache entry components.push(versionSalt); - return crypto - .createHash('sha256') - .update(components.join('|')) - .digest('hex'); + return crypto.createHash('sha256').update(components.join('|')).digest('hex'); } exports.getCacheVersion = getCacheVersion; function getCacheEntry(keys, paths, options) { @@ -740,13 +737,21 @@ function downloadCache(archiveLocation, archivePath, options) { return __awaiter(this, void 0, void 0, function* () { const archiveUrl = new url_1.URL(archiveLocation); const downloadOptions = (0, options_1.getDownloadOptions)(options); - if (downloadOptions.useAzureSdk && - archiveUrl.hostname.endsWith('.blob.core.windows.net')) { - // Use Azure storage SDK to download caches hosted on Azure to improve speed and reliability. - yield (0, downloadUtils_1.downloadCacheStorageSDK)(archiveLocation, archivePath, downloadOptions); + if (archiveUrl.hostname.endsWith('.blob.core.windows.net')) { + if (downloadOptions.useAzureSdk) { + // Use Azure storage SDK to download caches hosted on Azure to improve speed and reliability. + yield (0, downloadUtils_1.downloadCacheStorageSDK)(archiveLocation, archivePath, downloadOptions); + } + else if (downloadOptions.concurrentBlobDownloads) { + // Use concurrent implementation with HttpClient to work around blob SDK issue + yield (0, downloadUtils_1.downloadCacheHttpClientConcurrent)(archiveLocation, archivePath, downloadOptions); + } + else { + // Otherwise, download using the Actions http-client. + yield (0, downloadUtils_1.downloadCacheHttpClient)(archiveLocation, archivePath); + } } else { - // Otherwise, download using the Actions http-client. yield (0, downloadUtils_1.downloadCacheHttpClient)(archiveLocation, archivePath); } }); @@ -779,9 +784,7 @@ function getContentRange(start, end) { } function uploadChunk(httpClient, resourceUrl, openStream, start, end) { return __awaiter(this, void 0, void 0, function* () { - core.debug(`Uploading chunk of size ${end - - start + - 1} bytes at offset ${start} with content range: ${getContentRange(start, end)}`); + core.debug(`Uploading chunk of size ${end - start + 1} bytes at offset ${start} with content range: ${getContentRange(start, end)}`); const additionalHeaders = { 'Content-Type': 'application/octet-stream', 'Content-Range': getContentRange(start, end) @@ -949,35 +952,42 @@ function getArchiveFileSizeInBytes(filePath) { } exports.getArchiveFileSizeInBytes = getArchiveFileSizeInBytes; function resolvePaths(patterns) { - var e_1, _a; - var _b; + var _a, e_1, _b, _c; + var _d; return __awaiter(this, void 0, void 0, function* () { const paths = []; - const workspace = (_b = process.env['GITHUB_WORKSPACE']) !== null && _b !== void 0 ? _b : process.cwd(); + const workspace = (_d = process.env['GITHUB_WORKSPACE']) !== null && _d !== void 0 ? _d : process.cwd(); const globber = yield glob.create(patterns.join('\n'), { implicitDescendants: false }); try { - for (var _c = __asyncValues(globber.globGenerator()), _d; _d = yield _c.next(), !_d.done;) { - const file = _d.value; - const relativeFile = path - .relative(workspace, file) - .replace(new RegExp(`\\${path.sep}`, 'g'), '/'); - core.debug(`Matched: ${relativeFile}`); - // Paths are made relative so the tar entries are all relative to the root of the workspace. - if (relativeFile === '') { - // path.relative returns empty string if workspace and file are equal - paths.push('.'); + for (var _e = true, _f = __asyncValues(globber.globGenerator()), _g; _g = yield _f.next(), _a = _g.done, !_a;) { + _c = _g.value; + _e = false; + try { + const file = _c; + const relativeFile = path + .relative(workspace, file) + .replace(new RegExp(`\\${path.sep}`, 'g'), '/'); + core.debug(`Matched: ${relativeFile}`); + // Paths are made relative so the tar entries are all relative to the root of the workspace. + if (relativeFile === '') { + // path.relative returns empty string if workspace and file are equal + paths.push('.'); + } + else { + paths.push(`${relativeFile}`); + } } - else { - paths.push(`${relativeFile}`); + finally { + _e = true; } } } catch (e_1_1) { e_1 = { error: e_1_1 }; } finally { try { - if (_d && !_d.done && (_a = _c.return)) yield _a.call(_c); + if (!_e && !_a && (_b = _f.return)) yield _b.call(_f); } finally { if (e_1) throw e_1.error; } } @@ -1142,7 +1152,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge }); }; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.downloadCacheStorageSDK = exports.downloadCacheHttpClient = exports.DownloadProgress = void 0; +exports.downloadCacheStorageSDK = exports.downloadCacheHttpClientConcurrent = exports.downloadCacheHttpClient = exports.DownloadProgress = void 0; const core = __importStar(__nccwpck_require__(2186)); const http_client_1 = __nccwpck_require__(6255); const storage_blob_1 = __nccwpck_require__(4100); @@ -1299,6 +1309,115 @@ function downloadCacheHttpClient(archiveLocation, archivePath) { }); } exports.downloadCacheHttpClient = downloadCacheHttpClient; +/** + * Download the cache using the Actions toolkit http-client concurrently + * + * @param archiveLocation the URL for the cache + * @param archivePath the local path where the cache is saved + */ +function downloadCacheHttpClientConcurrent(archiveLocation, archivePath, options) { + var _a; + return __awaiter(this, void 0, void 0, function* () { + const archiveDescriptor = yield fs.promises.open(archivePath, 'w'); + const httpClient = new http_client_1.HttpClient('actions/cache', undefined, { + socketTimeout: options.timeoutInMs, + keepAlive: true + }); + try { + const res = yield (0, requestUtils_1.retryHttpClientResponse)('downloadCacheMetadata', () => __awaiter(this, void 0, void 0, function* () { return yield httpClient.request('HEAD', archiveLocation, null, {}); })); + const lengthHeader = res.message.headers['content-length']; + if (lengthHeader === undefined || lengthHeader === null) { + throw new Error('Content-Length not found on blob response'); + } + const length = parseInt(lengthHeader); + if (Number.isNaN(length)) { + throw new Error(`Could not interpret Content-Length: ${length}`); + } + const downloads = []; + const blockSize = 4 * 1024 * 1024; + for (let offset = 0; offset < length; offset += blockSize) { + const count = Math.min(blockSize, length - offset); + downloads.push({ + offset, + promiseGetter: () => __awaiter(this, void 0, void 0, function* () { + return yield downloadSegmentRetry(httpClient, archiveLocation, offset, count); + }) + }); + } + // reverse to use .pop instead of .shift + downloads.reverse(); + let actives = 0; + let bytesDownloaded = 0; + const progress = new DownloadProgress(length); + progress.startDisplayTimer(); + const progressFn = progress.onProgress(); + const activeDownloads = []; + let nextDownload; + const waitAndWrite = () => __awaiter(this, void 0, void 0, function* () { + const segment = yield Promise.race(Object.values(activeDownloads)); + yield archiveDescriptor.write(segment.buffer, 0, segment.count, segment.offset); + actives--; + delete activeDownloads[segment.offset]; + bytesDownloaded += segment.count; + progressFn({ loadedBytes: bytesDownloaded }); + }); + while ((nextDownload = downloads.pop())) { + activeDownloads[nextDownload.offset] = nextDownload.promiseGetter(); + actives++; + if (actives >= ((_a = options.downloadConcurrency) !== null && _a !== void 0 ? _a : 10)) { + yield waitAndWrite(); + } + } + while (actives > 0) { + yield waitAndWrite(); + } + } + finally { + httpClient.dispose(); + yield archiveDescriptor.close(); + } + }); +} +exports.downloadCacheHttpClientConcurrent = downloadCacheHttpClientConcurrent; +function downloadSegmentRetry(httpClient, archiveLocation, offset, count) { + return __awaiter(this, void 0, void 0, function* () { + const retries = 5; + let failures = 0; + while (true) { + try { + const timeout = 30000; + const result = yield promiseWithTimeout(timeout, downloadSegment(httpClient, archiveLocation, offset, count)); + if (typeof result === 'string') { + throw new Error('downloadSegmentRetry failed due to timeout'); + } + return result; + } + catch (err) { + if (failures >= retries) { + throw err; + } + failures++; + } + } + }); +} +function downloadSegment(httpClient, archiveLocation, offset, count) { + return __awaiter(this, void 0, void 0, function* () { + const partRes = yield (0, requestUtils_1.retryHttpClientResponse)('downloadCachePart', () => __awaiter(this, void 0, void 0, function* () { + return yield httpClient.get(archiveLocation, { + Range: `bytes=${offset}-${offset + count - 1}` + }); + })); + if (!partRes.readBodyBuffer) { + throw new Error('Expected HttpClientResponse to implement readBodyBuffer'); + } + return { + offset, + count, + buffer: yield partRes.readBodyBuffer() + }; + }); +} /** * Download the cache using the Azure Storage SDK. Only call this method if the * URL points to an Azure Storage endpoint. @@ -1864,7 +1983,8 @@ exports.getUploadOptions = getUploadOptions; */ function getDownloadOptions(copy) { const result = { - useAzureSdk: true, + useAzureSdk: false, + concurrentBlobDownloads: true, downloadConcurrency: 8, timeoutInMs: 30000, segmentTimeoutInMs: 600000, @@ -1874,6 +1994,9 @@ function getDownloadOptions(copy) { if (typeof copy.useAzureSdk === 'boolean') { result.useAzureSdk = copy.useAzureSdk; } + if (typeof copy.concurrentBlobDownloads === 'boolean') { + result.concurrentBlobDownloads = copy.concurrentBlobDownloads; + } if (typeof copy.downloadConcurrency === 'number') { result.downloadConcurrency = copy.downloadConcurrency; } @@ -5560,6 +5683,19 @@ class HttpClientResponse { })); }); } + readBodyBuffer() { + return __awaiter(this, void 0, void 0, function* () { + return new Promise((resolve) => __awaiter(this, void 0, void 0, function* () { + const chunks = []; + this.message.on('data', (chunk) => { + chunks.push(chunk); + }); + this.message.on('end', () => { + resolve(Buffer.concat(chunks)); + }); + })); + }); + } } exports.HttpClientResponse = HttpClientResponse; function isHttps(requestUrl) { @@ -6064,7 +6200,13 @@ function getProxyUrl(reqUrl) { } })(); if (proxyVar) { - return new URL(proxyVar); + try { + return new URL(proxyVar); + } + catch (_a) { + if (!proxyVar.startsWith('http://') && !proxyVar.startsWith('https://')) + return new URL(`http://${proxyVar}`); + } } else { return undefined; @@ -6075,6 +6217,10 @@ function checkBypass(reqUrl) { if (!reqUrl.hostname) { return false; } + const reqHost = reqUrl.hostname; + if (isLoopbackAddress(reqHost)) { + return true; + } const noProxy = process.env['no_proxy'] || process.env['NO_PROXY'] || ''; if (!noProxy) { return false; @@ -6100,13 +6246,24 @@ function checkBypass(reqUrl) { .split(',') .map(x => x.trim().toUpperCase()) .filter(x => x)) { - if (upperReqHosts.some(x => x === upperNoProxyItem)) { + if (upperNoProxyItem === '*' || + upperReqHosts.some(x => x === upperNoProxyItem || + x.endsWith(`.${upperNoProxyItem}`) || + (upperNoProxyItem.startsWith('.') && + x.endsWith(`${upperNoProxyItem}`)))) { return true; } } return false; } exports.checkBypass = checkBypass; +function isLoopbackAddress(host) { + const hostLower = host.toLowerCase(); + return (hostLower === 'localhost' || + hostLower.startsWith('127.') || + hostLower.startsWith('[::1]') || + hostLower.startsWith('[0:0:0:0:0:0:0:1]')); +} //# sourceMappingURL=proxy.js.map /***/ }), diff --git a/package-lock.json b/package-lock.json index 0f6fa02..7e776ec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "0.0.3", "license": "MIT", "dependencies": { - "@actions/cache": "^3.2.1", + "@actions/cache": "^3.2.2", "@actions/core": "^1.10.0", "@actions/exec": "^1.1.1", "@actions/tool-cache": "^2.0.1" @@ -42,14 +42,14 @@ } }, "node_modules/@actions/cache": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/@actions/cache/-/cache-3.2.1.tgz", - "integrity": "sha512-QurbMiY//02+0kN1adJkMHN44RcZ5kAXfhSnKUZmtSmhMTNqLitGArG1xOkt93NNyByTlLGAc5wIOF/dZ2ENOQ==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@actions/cache/-/cache-3.2.2.tgz", + "integrity": "sha512-6D0Jq5JrLZRQ3VApeQwQkkV20ZZXjXsHNYXd9VjNUdi9E0h93wESpxfMJ2JWLCUCgHNLcfY0v3GjNM+2FdRMlg==", "dependencies": { "@actions/core": "^1.10.0", "@actions/exec": "^1.0.1", "@actions/glob": "^0.1.0", - "@actions/http-client": "^2.0.1", + "@actions/http-client": "^2.1.1", "@actions/io": "^1.0.1", "@azure/abort-controller": "^1.1.0", "@azure/ms-rest-js": "^2.6.0", @@ -93,9 +93,9 @@ } }, "node_modules/@actions/http-client": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.0.1.tgz", - "integrity": "sha512-PIXiMVtz6VvyaRsGY268qvj57hXQEpsYogYOu2nrQhlf+XCGmZstmuZBbAybUl1nQGnvS1k1eEsQ69ZoD7xlSw==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.1.1.tgz", + "integrity": "sha512-qhrkRMB40bbbLo7gF+0vu+X+UawOvQQqNAA/5Unx774RS8poaOhThDOG6BGmxvAnxhQnDp2BG/ZUm65xZILTpw==", "dependencies": { "tunnel": "^0.0.6" } @@ -7507,14 +7507,14 @@ "dev": true }, "@actions/cache": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/@actions/cache/-/cache-3.2.1.tgz", - "integrity": "sha512-QurbMiY//02+0kN1adJkMHN44RcZ5kAXfhSnKUZmtSmhMTNqLitGArG1xOkt93NNyByTlLGAc5wIOF/dZ2ENOQ==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@actions/cache/-/cache-3.2.2.tgz", + "integrity": "sha512-6D0Jq5JrLZRQ3VApeQwQkkV20ZZXjXsHNYXd9VjNUdi9E0h93wESpxfMJ2JWLCUCgHNLcfY0v3GjNM+2FdRMlg==", "requires": { "@actions/core": "^1.10.0", "@actions/exec": "^1.0.1", "@actions/glob": "^0.1.0", - "@actions/http-client": "^2.0.1", + "@actions/http-client": "^2.1.1", "@actions/io": "^1.0.1", "@azure/abort-controller": "^1.1.0", "@azure/ms-rest-js": "^2.6.0", @@ -7557,9 +7557,9 @@ } }, "@actions/http-client": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.0.1.tgz", - "integrity": "sha512-PIXiMVtz6VvyaRsGY268qvj57hXQEpsYogYOu2nrQhlf+XCGmZstmuZBbAybUl1nQGnvS1k1eEsQ69ZoD7xlSw==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.1.1.tgz", + "integrity": "sha512-qhrkRMB40bbbLo7gF+0vu+X+UawOvQQqNAA/5Unx774RS8poaOhThDOG6BGmxvAnxhQnDp2BG/ZUm65xZILTpw==", "requires": { "tunnel": "^0.0.6" } diff --git a/package.json b/package.json index 06c8dc6..da7366d 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "author": "medyagh", "license": "MIT", "dependencies": { - "@actions/cache": "^3.2.1", + "@actions/cache": "^3.2.2", "@actions/core": "^1.10.0", "@actions/exec": "^1.1.1", "@actions/tool-cache": "^2.0.1"