From bcdaebf61d0536a1b9181bccc214a51b3649515b Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 12 Nov 2024 13:42:07 +0100 Subject: [PATCH] [OV JS] Make download binaries script universal (#27465) ### Details: - Implement **BinaryManager** class that configurable and universal - Extract helpers logic into utils - Make `download-binaries` script short and readable ### Tickets: - 156951 --------- Co-authored-by: Alicja Miloszewska --- src/bindings/js/node/package.json | 4 +- .../js/node/scripts/download-runtime.js | 24 ++ .../js/node/scripts/download_runtime.js | 302 ------------------ .../js/node/scripts/lib/binary-manager.js | 172 ++++++++++ src/bindings/js/node/scripts/lib/utils.js | 115 +++++++ src/bindings/js/node/tests/unit/utils.js | 2 +- 6 files changed, 314 insertions(+), 305 deletions(-) create mode 100644 src/bindings/js/node/scripts/download-runtime.js delete mode 100644 src/bindings/js/node/scripts/download_runtime.js create mode 100644 src/bindings/js/node/scripts/lib/binary-manager.js create mode 100644 src/bindings/js/node/scripts/lib/utils.js diff --git a/src/bindings/js/node/package.json b/src/bindings/js/node/package.json index 8bc6bbd4bb1d46..1ca1f10cdf57c2 100644 --- a/src/bindings/js/node/package.json +++ b/src/bindings/js/node/package.json @@ -3,7 +3,7 @@ "version": "2024.4.0", "description": "OpenVINO™ utils for using from Node.js environment", "repository": { - "url": "https://github.com/openvinotoolkit/openvino.git", + "url": "git+https://github.com/openvinotoolkit/openvino.git", "type": "git" }, "license": "Apache-2.0", @@ -23,7 +23,7 @@ "test:e2e": "mocha ./tests/e2e/electron-app.test.js", "tsc": "tsc", "postinstall": "npm run install_runtime", - "download_runtime": "node ./scripts/download_runtime.js", + "download_runtime": "node ./scripts/download-runtime.js", "install_runtime": "npm run download_runtime -- --ignore-if-exists" }, "devDependencies": { diff --git a/src/bindings/js/node/scripts/download-runtime.js b/src/bindings/js/node/scripts/download-runtime.js new file mode 100644 index 00000000000000..90bece67161a6a --- /dev/null +++ b/src/bindings/js/node/scripts/download-runtime.js @@ -0,0 +1,24 @@ +const { join } = require('node:path'); + +const BinaryManager = require('./lib/binary-manager'); +const packageJson = require('../package.json'); + +if (require.main === module) main(); + +async function main() { + if (!BinaryManager.isCompatible()) process.exit(1); + + const force = process.argv.includes('-f') || process.argv.includes('--force'); + const ignoreIfExists = process.argv.includes('-i') + || process.argv.includes('--ignore-if-exists'); + + const { env } = process; + const proxy = env.http_proxy || env.HTTP_PROXY || env.npm_config_proxy; + + await BinaryManager.prepareBinary( + join(__dirname, '..'), + packageJson.version, + packageJson.binary, + { force, ignoreIfExists, proxy }, + ); +} diff --git a/src/bindings/js/node/scripts/download_runtime.js b/src/bindings/js/node/scripts/download_runtime.js deleted file mode 100644 index 321eb4b125bc6c..00000000000000 --- a/src/bindings/js/node/scripts/download_runtime.js +++ /dev/null @@ -1,302 +0,0 @@ -const os = require('os'); -const path = require('path'); -const tar = require('tar-fs'); -const https = require('node:https'); -const gunzip = require('gunzip-maybe'); -const fs = require('node:fs/promises'); -const { createReadStream, createWriteStream } = require('node:fs'); -const { HttpsProxyAgent } = require('https-proxy-agent'); - -const packageJson = require('../package.json'); - -const codeENOENT = 'ENOENT'; - -if (require.main === module) { - main(); -} - -async function main() { - const modulePath = packageJson.binary['module_path']; - const destinationPath = path.resolve(__dirname, '..', modulePath); - const force = process.argv.includes('-f'); - const ignoreIfExists = process.argv.includes('--ignore-if-exists'); - const { env } = process; - const proxy = env.http_proxy || env.HTTP_PROXY || env.npm_config_proxy; - - try { - await downloadRuntime(destinationPath, { force, ignoreIfExists, proxy }); - } catch(error) { - if (error instanceof RuntimeExistsError) { - console.error( - `Directory '${destinationPath}' already exists. ` + - 'To force runtime downloading run \'npm run download_runtime -- -f\'', - ); - } else { - throw error; - } - process.exit(1); - } -} - -class RuntimeExistsError extends Error { - constructor(message) { - super(message); - this.name = 'RuntimeExistsError'; - Error.captureStackTrace(this, RuntimeExistsError); - } -} - -/** - * Download OpenVINO Runtime archive and extract it to destination directory. - * - * @async - * @function downloadRuntime - * @param {string} destinationPath - The destination directory path. - * @param {Object} [config] - The configuration object. - * @param {boolean} [config.force=false] - The flag - * to force install and replace runtime if it exists. Default is `false`. - * @param {boolean} [config.ignoreIfExists=true] - The flag - * to skip installation if it exists Default is `true`. - * @param {string|null} [config.proxy=null] - The proxy URL. Default is `null`. - * @returns {Promise} - * @throws {RuntimeExistsError} - */ -async function downloadRuntime( - destinationPath, - config = { force: false, ignoreIfExists: true, proxy: null }, -) { - const { version } = packageJson; - const osInfo = await getOsInfo(); - const isRuntimeDirectoryExists = await checkIfPathExists(destinationPath); - - if (isRuntimeDirectoryExists && !config.force) { - if (config.ignoreIfExists) { - console.warn( - `Directory '${destinationPath}' already exists. Skipping ` + - 'runtime downloading because \'ignoreIfExists\' flag is passed.', - ); - - return; - } - - throw new RuntimeExistsError( - `Directory '${destinationPath}' already exists. ` + - 'To force runtime downloading use \'force\' flag.', - ); - } - - const runtimeArchiveUrl = getRuntimeArchiveUrl(version, osInfo); - const tmpDir = `temp-ov-runtime-archive-${new Date().getTime()}`; - const tempDirectoryPath = path.join(os.tmpdir(), tmpDir); - - try { - const filename = path.basename(runtimeArchiveUrl); - const archiveFilePath = path.resolve(tempDirectoryPath, filename); - - await fs.mkdir(tempDirectoryPath); - - console.log('Downloading OpenVINO runtime archive...'); - await downloadFile( - runtimeArchiveUrl, - tempDirectoryPath, - filename, - config.proxy, - ); - console.log('OpenVINO runtime archive downloaded.'); - - await removeDirectory(destinationPath); - - console.log('Extracting archive...'); - await unarchive(archiveFilePath, destinationPath); - - console.log('The archive was successfully extracted.'); - } catch(error) { - console.error(`Failed to download OpenVINO runtime: ${error}.`); - throw error; - } finally { - await removeDirectory(tempDirectoryPath); - } -} - -/** - * The OS information object. - * @typedef {Object} OsInfo - * @property {NodeJS.Platform} platform - * @property {string} arch - */ - -/** - * Get information about OS. - * - * @async - * @function getOsInfo - * @returns {Promise} - */ -async function getOsInfo() { - const platform = os.platform(); - - if (!['win32', 'linux', 'darwin'].includes(platform)) { - throw new Error(`Platform '${platform}' is not supported.`); - } - - const arch = os.arch(); - - if (!['arm64', 'armhf', 'x64'].includes(arch)) { - throw new Error(`Architecture '${arch}' is not supported.`); - } - - if (platform === 'win32' && arch !== 'x64') { - throw new Error(`Version for windows and '${arch}' is not supported.`); - } - - return { platform, arch }; -} - -/** - * Check if path exists. - * - * @async - * @function checkIfPathExists - * @param {string} path - The path to directory or file. - * @returns {Promise} - */ -async function checkIfPathExists(path) { - try { - await fs.access(path); - - return true; - } catch(error) { - if (error.code === codeENOENT) { - return false; - } - throw error; - } -} - -/** - * Get OpenVINO runtime archive URL. - * - * @function getRuntimeArchiveUrl - * @param {string} version - Package version. - * @param {OsInfo} osInfo - The OS related data. - * @returns {string} - */ -function getRuntimeArchiveUrl(version, osInfo) { - const { - host, - package_name: packageNameTemplate, - remote_path: remotePathTemplate, - } = packageJson.binary; - const fullPathTemplate = `${remotePathTemplate}${packageNameTemplate}`; - const fullPath = fullPathTemplate - .replace(new RegExp('{version}', 'g'), version) - .replace(new RegExp('{platform}', 'g'), osInfo.platform) - .replace(new RegExp('{arch}', 'g'), osInfo.arch); - - return new URL(fullPath, host).toString(); -} - -/** - * Remove directory and its content. - * - * @async - * @function removeDirectory - * @param {string} path - The directory path. - * @returns {Promise} - */ -async function removeDirectory(path) { - try { - console.log(`Removing ${path}`); - await fs.rm(path, { recursive: true, force: true }); - } catch(error) { - if (error.code === codeENOENT) console.log(`Path: ${path} doesn't exist`); - - throw error; - } -} - -/** - * Download file by URL and save it to the destination path. - * - * @function downloadFile - * @param {string} url - The file URL. - * @param {string} filename - The filename of result file. - * @param {string} destination - The destination path of result file. - * @param {string} [proxy=null] - (Optional) The proxy URL. - * @returns {Promise} - */ -function downloadFile(url, destination, filename, proxy = null) { - const timeout = 5000; - const fullPath = path.resolve(destination, filename); - const file = createWriteStream(fullPath); - - if (new URL(url).protocol === 'http') - throw new Error('Http link doesn\'t support'); - - let agent; - - if (proxy) { - agent = new HttpsProxyAgent(proxy); - console.log(`Proxy agent is configured with '${proxy}'.`); - } - - return new Promise((resolve, reject) => { - file.on('error', (error) => { - reject(`Failed to open file stream: ${error}.`); - }); - - console.log(`Download file by link: ${url}`); - - const request = https.get(url, { agent }, (res) => { - const { statusCode } = res; - - if (statusCode !== 200) { - return reject(`Server returned status code ${statusCode}.`); - } - - res.pipe(file); - - file.on('finish', () => { - file.close(); - console.log(`File was successfully downloaded to '${fullPath}'.`); - resolve(); - }); - }); - - request.on('error', (error) => { - reject(`Failed to send request: ${error}.`); - }); - - request.setTimeout(timeout, () => { - request.destroy(); - reject(`Request was timed out after ${timeout} ms.`); - }); - }); -} - -/** - * Unarchive tar and tar.gz archives. - * - * @function unarchive - * @param {tarFilePath} tarFilePath - Path to archive. - * @param {dest} tarFilePath - Path where to unpack. - * @returns {Promise} - */ -function unarchive(tarFilePath, dest) { - return new Promise((resolve, reject) => { - createReadStream(tarFilePath) - .pipe(gunzip()) - .pipe( - tar - .extract(dest) - .on('finish', () => { - resolve(); - }) - .on('error', (err) => { - reject(err); - }), - ); - }); -} - -module.exports = { downloadRuntime, downloadFile, checkIfPathExists }; diff --git a/src/bindings/js/node/scripts/lib/binary-manager.js b/src/bindings/js/node/scripts/lib/binary-manager.js new file mode 100644 index 00000000000000..f0af78b49093ec --- /dev/null +++ b/src/bindings/js/node/scripts/lib/binary-manager.js @@ -0,0 +1,172 @@ +const os = require('node:os'); +const tar = require('tar-fs'); +const path = require('node:path'); +const gunzip = require('gunzip-maybe'); +const fs = require('node:fs/promises'); +const { createReadStream } = require('node:fs'); + +const { downloadFile, checkIfPathExists, removeDirectory } = require('./utils'); + +class BinaryManager { + constructor(packageRoot, version, binaryConfig) { + this.packageRoot = packageRoot; + this.version = version; + this.binaryConfig = binaryConfig; + } + + getPlatformLabel() { + return os.platform(); + } + + getArchLabel() { + return os.arch(); + } + + getExtension() { + return 'tar.gz'; + } + + getArchiveUrl() { + const { + host, + package_name: packageNameTemplate, + remote_path: remotePathTemplate, + } = this.binaryConfig; + const fullPathTemplate = `${remotePathTemplate}${packageNameTemplate}` + const fullPath = fullPathTemplate + .replace(new RegExp('{version}', 'g'), this.version) + .replace(new RegExp('{platform}', 'g'), this.getPlatformLabel()) + .replace(new RegExp('{arch}', 'g'), this.getArchLabel()) + .replace(new RegExp('{extension}', 'g'), this.getExtension()); + + return new URL(fullPath, host).toString(); + } + + getDestinationPath() { + const modulePath = this.binaryConfig['module_path']; + + return path.resolve(this.packageRoot, modulePath); + } + + /** + * Prepares the binary by downloading and extracting the OpenVINO runtime archive. + * + * @param {string} packageRoot - The root directory of the package. + * @param {string} version - The version of the binary. + * @param {Object} binaryConfig - The configuration object for the binary. + * @param {Object} options - The options for preparing the binary. + * @param {boolean} options.force - Whether to force the download if the directory already exists. + * @param {boolean} options.ignoreIfExists - Whether to ignore the download if the directory already exists. + * @param {string} [options.proxy] - The proxy to use for downloading the file. + * @throws {Error} If the directory already exists and the force option is not set. + * @throws {Error} If the download or extraction fails. + * @returns {Promise} A promise that resolves when the binary is prepared. + */ + static async prepareBinary(packageRoot, version, binaryConfig, options) { + const binaryManager = new this(packageRoot, version, binaryConfig); + const destinationPath = binaryManager.getDestinationPath(); + const isRuntimeDirectoryExists = await checkIfPathExists(destinationPath); + + if (isRuntimeDirectoryExists && !options.force) { + if (options.ignoreIfExists) { + console.warn( + `Directory '${destinationPath}' already exists. Skipping ` + + 'runtime downloading because "ignoreIfExists" flag is passed.' + ); + + return; + } + + throw new Error( + `Directory '${destinationPath}' already exists. ` + + 'To force runtime downloading use "force" flag.', + ); + } + + const archiveUrl = binaryManager.getArchiveUrl(); + let tempDirectoryPath = null; + + try { + tempDirectoryPath = await fs.mkdtemp( + path.join(os.tmpdir(), 'temp-ov-runtime-archive-') + ); + + const filename = path.basename(archiveUrl); + + console.log('Downloading OpenVINO runtime archive...'); + const archiveFilePath = await downloadFile( + archiveUrl, + tempDirectoryPath, + filename, + options.proxy, + ) + console.log('OpenVINO runtime archive downloaded.'); + + await removeDirectory(destinationPath); + await this.unarchive(archiveFilePath, destinationPath); + console.log('The archive was successfully extracted.'); + } catch(error) { + console.error(`Failed to download OpenVINO runtime: ${error}.`); + throw error; + } finally { + if (tempDirectoryPath) await removeDirectory(tempDirectoryPath); + } + } + + /** + * Checks if the current platform and architecture are compatible. + * + * Supported platforms: 'win32', 'linux', 'darwin'. + * Supported architectures: 'arm64', 'armhf', 'x64'. + * + * If the platform or architecture is not supported, an error message is logged to the console. + * + * @returns {boolean} Returns true if the platform and architecture are compatible, otherwise false. + */ + static isCompatible() { + const missleadings = []; + const platform = os.platform(); + + if (!['win32', 'linux', 'darwin'].includes(platform)) + missleadings.push(`Platform '${platform}' is not supported.`); + + const arch = os.arch(); + + if (!['arm64', 'armhf', 'x64'].includes(arch)) + missleadings.push(`Architecture '${arch}' is not supported.`); + + if (platform === 'win32' && arch !== 'x64') + missleadings.push(`Version for windows and '${arch}' is not supported.`); + + if (missleadings.length) { + console.error(missleadings.join(' ')); + return false; + } + + return true; + } + + /** + * Unarchive tar and tar.gz archives. + * + * @function unarchive + * @param {string} archivePath - Path to archive. + * @param {string} dest - Path where to unpack. + * @returns {Promise} + */ + static unarchive(archivePath, dest) { + return new Promise((resolve, reject) => { + createReadStream(archivePath) + .pipe(gunzip()) + .pipe(tar.extract(dest) + .on('finish', () => { + resolve(); + }).on('error', (err) => { + reject(err); + }), + ); + }); + } +} + +module.exports = BinaryManager; diff --git a/src/bindings/js/node/scripts/lib/utils.js b/src/bindings/js/node/scripts/lib/utils.js new file mode 100644 index 00000000000000..9658ec504fa0d9 --- /dev/null +++ b/src/bindings/js/node/scripts/lib/utils.js @@ -0,0 +1,115 @@ +const path = require('node:path'); +const https = require('node:https'); +const fs = require('node:fs/promises'); +const { createWriteStream } = require('node:fs'); + +const { HttpsProxyAgent } = require('https-proxy-agent'); + +const codeENOENT = 'ENOENT'; + +module.exports = { + removeDirectory, + checkIfPathExists, + downloadFile, +}; + +/** + * Remove directory and its content. + * + * @async + * @function removeDirectory + * @param {string} path - The directory path. + * @returns {Promise} + */ +async function removeDirectory(path) { + try { + console.log(`Removing ${path}`); + await fs.rm(path, { recursive: true }); + } catch (error) { + if (error.code !== codeENOENT) throw error; + + console.warn(`Path: ${path} doesn't exist`); + } +} + +/** + * Check if path exists. + * + * @async + * @function checkIfPathExists + * @param {string} path - The path to directory or file. + * @returns {Promise} + */ +async function checkIfPathExists(path) { + try { + await fs.access(path); + return true; + } catch (error) { + if (error.code === codeENOENT) { + return false; + } + throw error; + } +} + +/** + * Download file by URL and save it to the destination path. + * + * @function downloadFile + * @param {string} url - The file URL. + * @param {string} filename - The filename of result file. + * @param {string} destination - The destination path of result file. + * @param {string} [proxy=null] - (Optional) The proxy URL. + * @returns {Promise} - Path to downloaded file. + */ +function downloadFile(url, destination, filename, proxy = null) { + console.log(`Downloading file by link: ${url} to ${destination}` + + `with filename: ${filename}`); + + const timeout = 5000; + const fullPath = path.resolve(destination, filename); + const file = createWriteStream(fullPath); + + if (new URL(url).protocol === 'http:') + throw new Error('Http link doesn\'t support'); + + let agent; + + if (proxy) { + agent = new HttpsProxyAgent(proxy); + console.log(`Proxy agent is configured with '${proxy}'.`); + } + + return new Promise((resolve, reject) => { + file.on('error', (error) => { + reject(`Failed to open file stream: ${error}.`); + }); + + console.log(`Download file by link: ${url}`); + + const request = https.get(url, { agent }, (res) => { + const { statusCode } = res; + + if (statusCode !== 200) { + return reject(`Server returned status code ${statusCode}.`); + } + + res.pipe(file); + + file.on('finish', () => { + file.close(); + console.log(`File was successfully downloaded to '${fullPath}'.`); + resolve(fullPath); + }); + }); + + request.on('error', (error) => { + reject(`Failed to send request: ${error}.`); + }); + + request.setTimeout(timeout, () => { + request.destroy(); + reject(`Request was timed out after ${timeout} ms.`); + }); + }); +} diff --git a/src/bindings/js/node/tests/unit/utils.js b/src/bindings/js/node/tests/unit/utils.js index c2089e30b4cdc8..456f87983dba20 100644 --- a/src/bindings/js/node/tests/unit/utils.js +++ b/src/bindings/js/node/tests/unit/utils.js @@ -7,7 +7,7 @@ const fs = require('node:fs/promises'); const { downloadFile, checkIfPathExists, -} = require('../../scripts/download_runtime'); +} = require('../../scripts/lib/utils'); const modelDir = 'tests/unit/test_models/'; const testModels = {