From 0d7375745309f8b17e646f6daff01f71c187b066 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 11 Nov 2024 17:53:00 +0100 Subject: [PATCH 01/47] Add genai nodejs bindings --- samples/js/.gitignore | 1 + samples/js/app.js | 28 ++++ samples/js/interactive.js | 56 +++++++ samples/js/package-lock.json | 37 +++++ samples/js/package.json | 12 ++ src/CMakeLists.txt | 4 + src/js/.gitignore | 5 + src/js/.npmignore | 15 ++ src/js/CMakeLists.txt | 92 +++++++++++ src/js/README.md | 31 ++++ src/js/include/addon.hpp | 20 +++ src/js/include/helper.hpp | 23 +++ .../llm_pipeline/finish_chat_worker.hpp | 18 +++ src/js/include/llm_pipeline/init_worker.hpp | 21 +++ .../llm_pipeline/llm_pipeline_wrapper.hpp | 27 ++++ .../llm_pipeline/start_chat_worker.hpp | 18 +++ src/js/lib/bindings.cjs | 1 + src/js/lib/module.js | 118 ++++++++++++++ src/js/package-lock.json | 21 +++ src/js/package.json | 24 +++ src/js/src/addon.cpp | 30 ++++ src/js/src/helper.cpp | 53 ++++++ .../src/llm_pipeline/finish_chat_worker.cpp | 14 ++ src/js/src/llm_pipeline/init_worker.cpp | 18 +++ .../src/llm_pipeline/llm_pipeline_wrapper.cpp | 151 ++++++++++++++++++ src/js/src/llm_pipeline/start_chat_worker.cpp | 14 ++ src/js/tests/bindings.test.js | 71 ++++++++ src/js/tests/module.test.js | 140 ++++++++++++++++ src/js/tests/setup.js | 9 ++ src/js/tests/utils.js | 73 +++++++++ 30 files changed, 1145 insertions(+) create mode 100644 samples/js/.gitignore create mode 100644 samples/js/app.js create mode 100644 samples/js/interactive.js create mode 100644 samples/js/package-lock.json create mode 100644 samples/js/package.json create mode 100644 src/js/.gitignore create mode 100644 src/js/.npmignore create mode 100644 src/js/CMakeLists.txt create mode 100644 src/js/README.md create mode 100644 src/js/include/addon.hpp create mode 100644 src/js/include/helper.hpp create mode 100644 src/js/include/llm_pipeline/finish_chat_worker.hpp create mode 100644 src/js/include/llm_pipeline/init_worker.hpp create mode 100644 src/js/include/llm_pipeline/llm_pipeline_wrapper.hpp create mode 100644 src/js/include/llm_pipeline/start_chat_worker.hpp create mode 100644 src/js/lib/bindings.cjs create mode 100644 src/js/lib/module.js create mode 100644 src/js/package-lock.json create mode 100644 src/js/package.json create mode 100644 src/js/src/addon.cpp create mode 100644 src/js/src/helper.cpp create mode 100644 src/js/src/llm_pipeline/finish_chat_worker.cpp create mode 100644 src/js/src/llm_pipeline/init_worker.cpp create mode 100644 src/js/src/llm_pipeline/llm_pipeline_wrapper.cpp create mode 100644 src/js/src/llm_pipeline/start_chat_worker.cpp create mode 100644 src/js/tests/bindings.test.js create mode 100644 src/js/tests/module.test.js create mode 100644 src/js/tests/setup.js create mode 100644 src/js/tests/utils.js diff --git a/samples/js/.gitignore b/samples/js/.gitignore new file mode 100644 index 0000000000..3c3629e647 --- /dev/null +++ b/samples/js/.gitignore @@ -0,0 +1 @@ +node_modules diff --git a/samples/js/app.js b/samples/js/app.js new file mode 100644 index 0000000000..f57e2fad1d --- /dev/null +++ b/samples/js/app.js @@ -0,0 +1,28 @@ +import { Pipeline } from 'genai-node'; + +const MODEL_PATH = process.argv[2]; + +if (!MODEL_PATH) { + console.error('Please specify path to model directory\n' + + 'Run command must be: `node app.js *path_to_model_dir*`'); + process.exit(1); +} + +const generationCallback = (chunk) => { + process.stdout.write(chunk); +}; + +const prompt = 'Who are you?'; +console.log(`User Prompt: "${prompt}"\n`); + +const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH); + +await pipeline.startChat(); +const result = await pipeline.generate( + prompt, + generationCallback, + { temperature: 0 }, +); +await pipeline.finishChat(); + +console.log(`\n\nGeneration result:\n"${result}"`) diff --git a/samples/js/interactive.js b/samples/js/interactive.js new file mode 100644 index 0000000000..466326ced0 --- /dev/null +++ b/samples/js/interactive.js @@ -0,0 +1,56 @@ +import path from 'node:path'; +import readline from 'readline'; + +import { Pipeline } from 'genai-node'; + +const MODEL_PATH = process.argv[2]; + +if (!MODEL_PATH) { + console.error('Please specify path to model directory\n' + + 'Run command must be: `node app.js *path_to_model_dir*`'); + process.exit(1); +} + +main(); + +async function main() { + // Create interface for reading user input from stdin + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + + console.log(`Welcome! Model "${path.parse(MODEL_PATH).name}" loaded. ` + + 'Type something and press enter. Type "finish" to exit.'); + const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH); + await pipeline.startChat(); + promptUser(); + + // Function to prompt the user for input + function promptUser() { + rl.question('> ', handleInput); + } + + // Function to handle user input + async function handleInput(input) { + input = input.trim(); + + // Check for exit command + if (input === 'finish') { + console.log('Goodbye!'); + await pipeline.finishChat(); + rl.close(); + process.exit(0); + } + + const result = await pipeline.generate(input, generationCallback); + console.log('\n'); + + // Wait for new input + promptUser(); + } + + function generationCallback(chunk) { + process.stdout.write(chunk); + } +} diff --git a/samples/js/package-lock.json b/samples/js/package-lock.json new file mode 100644 index 0000000000..4b2d14913f --- /dev/null +++ b/samples/js/package-lock.json @@ -0,0 +1,37 @@ +{ + "name": "genai-node-demo", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "genai-node-demo", + "version": "1.0.0", + "license": "Apache-2.0", + "devDependencies": { + "genai-node": "../../src/js/" + }, + "engines": { + "node": ">=21.0.0" + } + }, + "../../src/js": { + "name": "genai-node", + "version": "2024.5.0-preview", + "dev": true, + "license": "Apache-2.0", + "os": [ + "linux", + "darwin", + "win32" + ], + "engines": { + "node": ">=21.0.0" + } + }, + "node_modules/genai-node": { + "resolved": "../../src/js", + "link": true + } + } +} diff --git a/samples/js/package.json b/samples/js/package.json new file mode 100644 index 0000000000..0251bdfa14 --- /dev/null +++ b/samples/js/package.json @@ -0,0 +1,12 @@ +{ + "name": "genai-node-demo", + "version": "1.0.0", + "license": "Apache-2.0", + "type": "module", + "devDependencies": { + "genai-node": "../../src/js/" + }, + "engines": { + "node": ">=21.0.0" + } +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d9f3cc64db..432d68cbef 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,3 +7,7 @@ add_subdirectory(cpp) if(ENABLE_PYTHON) add_subdirectory(python) endif() + +if(ENABLE_JS) + add_subdirectory(js) +endif() diff --git a/src/js/.gitignore b/src/js/.gitignore new file mode 100644 index 0000000000..e7fffbc37a --- /dev/null +++ b/src/js/.gitignore @@ -0,0 +1,5 @@ +.vscode +bin +bin.* +build +thirdparty diff --git a/src/js/.npmignore b/src/js/.npmignore new file mode 100644 index 0000000000..9bf3e571b1 --- /dev/null +++ b/src/js/.npmignore @@ -0,0 +1,15 @@ +.vscode +bin.* +build +include +src +tests + +.eslintrc.js +CMakeLists.txt +tsconfig.json +TODO.md +build.sh + +**/*.tsbuildinfo +*.tgz diff --git a/src/js/CMakeLists.txt b/src/js/CMakeLists.txt new file mode 100644 index 0000000000..96ba0019c1 --- /dev/null +++ b/src/js/CMakeLists.txt @@ -0,0 +1,92 @@ +cmake_minimum_required(VERSION 3.18) + +# Set C++ standard +set(CMAKE_CXX_STANDARD 17) + +set(dist_folder "${CMAKE_SOURCE_DIR}/bin/") + +if(WIN32) + set(CMAKE_SHARED_LINKER_FLAGS /DELAYLOAD:NODE.EXE) + set(CMAKE_JS_LIB ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/node.lib) + set(CMAKE_JS_SRC ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/win_delay_load_hook.cc) + + set(CMAKE_JS_NODELIB_DEF ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/node-lib.def) + set(CMAKE_JS_NODELIB_TARGET ${CMAKE_JS_LIB}) + set(DELAYIMP_LIB delayimp.lib) +endif() + +project(genai_node_addon) + +# Add definitions +add_definitions(-DNAPI_VERSION=8) + +include(FetchContent) + +FetchContent_Declare( + node-api-headers + URL https://github.com/nodejs/node-api-headers/archive/refs/tags/v1.1.0.tar.gz + URL_HASH SHA256=70608bc1e6dddce280285f3462f18a106f687c0720a4b90893e1ecd86e5a8bbf +) +FetchContent_MakeAvailable(node-api-headers) + +FetchContent_Declare( + node-addon-api + URL https://github.com/nodejs/node-addon-api/archive/refs/tags/v8.0.0.tar.gz + URL_HASH SHA256=42424c5206b9d67b41af4fcff5d6e3cb22074168035a03b8467852938a281d47 +) +FetchContent_MakeAvailable(node-addon-api) + +# Create a library +add_library(${PROJECT_NAME} SHARED + ${CMAKE_CURRENT_SOURCE_DIR}/src/addon.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/llm_pipeline/llm_pipeline_wrapper.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/llm_pipeline/finish_chat_worker.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/llm_pipeline/start_chat_worker.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/llm_pipeline/init_worker.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/helper.cpp + + ${CMAKE_JS_SRC} +) + +# Include directories +target_include_directories(${PROJECT_NAME} PRIVATE + "${node-api-headers_SOURCE_DIR}/include" + "${node-addon-api_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}" +) + +target_link_libraries(${PROJECT_NAME} PRIVATE openvino::genai ${DELAYIMP_LIB} ${CMAKE_JS_LIB}) + +if(MSVC AND CMAKE_JS_NODELIB_DEF AND CMAKE_JS_NODELIB_TARGET) # Generate node.lib + execute_process(COMMAND ${CMAKE_AR} /def:${CMAKE_JS_NODELIB_DEF} /out:${CMAKE_JS_NODELIB_TARGET} ${CMAKE_STATIC_LINKER_FLAGS}) +endif() + +if(APPLE) + target_link_options(${PROJECT_NAME} PRIVATE -Wl,-undefined,suppress,-flat_namespace) +elseif(AARCH64 OR ARM) + target_link_options(${PROJECT_NAME} PRIVATE -Wl,--unresolved-symbols=ignore-all) +endif() + +# Set library properties +set_target_properties(${PROJECT_NAME} PROPERTIES + PREFIX "" + SUFFIX ".node" +) + +# setting RPATH / LC_RPATH depending on platform +if(LINUX) + # to find libopenvino_genai.so in the same folder + set(rpaths "$ORIGIN") +elseif(APPLE) + # to find libopenvino_genai.dylib in the same folder + set(rpaths "@loader_path") +endif() + +if(rpaths) + set_target_properties(${PROJECT_NAME} PROPERTIES INSTALL_RPATH "${rpaths}") +endif() + +install(TARGETS ${PROJECT_NAME} + LIBRARY DESTINATION openvino_genai COMPONENT ${PROJECT_NAME} EXCLUDE_FROM_ALL + RUNTIME DESTINATION openvino_genai COMPONENT ${PROJECT_NAME} EXCLUDE_FROM_ALL +) diff --git a/src/js/README.md b/src/js/README.md new file mode 100644 index 0000000000..77ebc0792c --- /dev/null +++ b/src/js/README.md @@ -0,0 +1,31 @@ +# OpenVINO™ GenAI Node.js bindings (preview) + +## DISCLAIMER + +This is preview version, do not use it in production! + +## Install and Run + +### Requirements + +- Node.js v21+ +- Tested on Ubuntu, another OS didn't tested yet + +### Build Bindings + +TODO: Add instructions + +### Perform Test Run + +- To run sample you should have prepared model. + Use this instruction [to download model](https://github.com/openvinotoolkit/openvino.genai/blob/master/samples/cpp/chat_sample/README.md#download-and-convert-the-model-and-tokenizers) +- Go to [samples](../../samples/js/) +- Run `node app.js`, you should see: `User Prompt: ...` + +### Using as npm Dependency + +To use this package locally use `npm link` in this directory +and `npm link genai-node` in the folder where you want add this package as dependency + +To extract this package and use it as distributed npm package run `npm package`. +This command creates archive that you may use in your projects. diff --git a/src/js/include/addon.hpp b/src/js/include/addon.hpp new file mode 100644 index 0000000000..35e5cc462e --- /dev/null +++ b/src/js/include/addon.hpp @@ -0,0 +1,20 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +typedef Napi::Function (*Prototype)(Napi::Env); + +struct AddonData { + Napi::FunctionReference core; +}; + +void init_class(Napi::Env env, + Napi::Object exports, + std::string class_name, + Prototype func, + Napi::FunctionReference& reference); + +Napi::Object init_module(Napi::Env env, Napi::Object exports); diff --git a/src/js/include/helper.hpp b/src/js/include/helper.hpp new file mode 100644 index 0000000000..4a010df019 --- /dev/null +++ b/src/js/include/helper.hpp @@ -0,0 +1,23 @@ +#pragma once +#include + +#include "openvino/core/type/element_type.hpp" +#include "openvino/openvino.hpp" + +ov::AnyMap to_anyMap(const Napi::Env&, const Napi::Value&); + +/** + * @brief Template function to convert Javascript data types into C++ data types + * @tparam TargetType destinated C++ data type + * @param info Napi::CallbackInfo contains all arguments passed to a function or method + * @param idx specifies index of a argument inside info. + * @return specified argument converted to a TargetType. + */ +template +TargetType js_to_cpp(const Napi::Env& env, const Napi::Value& value); + +/** @brief A template specialization for TargetType ov::Any */ +template <> +ov::Any js_to_cpp(const Napi::Env& env, const Napi::Value& value); + +bool is_napi_value_int(const Napi::Env& env, const Napi::Value& num); diff --git a/src/js/include/llm_pipeline/finish_chat_worker.hpp b/src/js/include/llm_pipeline/finish_chat_worker.hpp new file mode 100644 index 0000000000..ca80b30aff --- /dev/null +++ b/src/js/include/llm_pipeline/finish_chat_worker.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include "openvino/genai/llm_pipeline.hpp" + +using namespace Napi; + +class FinishChatWorker : public AsyncWorker { + public: + FinishChatWorker(Function& callback, std::shared_ptr& pipe); + virtual ~FinishChatWorker(){}; + + void Execute(); + void OnOK(); + + private: + std::shared_ptr& pipe; +}; diff --git a/src/js/include/llm_pipeline/init_worker.hpp b/src/js/include/llm_pipeline/init_worker.hpp new file mode 100644 index 0000000000..5fc05969fb --- /dev/null +++ b/src/js/include/llm_pipeline/init_worker.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include +#include "openvino/genai/llm_pipeline.hpp" + +using namespace Napi; + +class InitWorker : public AsyncWorker { + public: + InitWorker(Function& callback, std::shared_ptr& pipe, + const std::string model_path, std::string device); + virtual ~InitWorker(){}; + + void Execute(); + void OnOK(); + + private: + std::shared_ptr& pipe; + std::string model_path; + std::string device; +}; diff --git a/src/js/include/llm_pipeline/llm_pipeline_wrapper.hpp b/src/js/include/llm_pipeline/llm_pipeline_wrapper.hpp new file mode 100644 index 0000000000..872e9ea023 --- /dev/null +++ b/src/js/include/llm_pipeline/llm_pipeline_wrapper.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include "openvino/genai/llm_pipeline.hpp" + +class LLMPipelineWrapper : public Napi::ObjectWrap { +public: + LLMPipelineWrapper(const Napi::CallbackInfo& info); + + static Napi::Function get_class(Napi::Env env); + + Napi::Value init(const Napi::CallbackInfo& info); + Napi::Value generate(const Napi::CallbackInfo& info); + Napi::Value start_chat(const Napi::CallbackInfo& info); + Napi::Value finish_chat(const Napi::CallbackInfo& info); +private: + bool is_loaded = false; + bool is_initialized = false; + bool is_running = false; + + std::string model_path; + std::string device; + + std::shared_ptr pipe = nullptr; + std::function streamer; +}; diff --git a/src/js/include/llm_pipeline/start_chat_worker.hpp b/src/js/include/llm_pipeline/start_chat_worker.hpp new file mode 100644 index 0000000000..fde0cfaa0a --- /dev/null +++ b/src/js/include/llm_pipeline/start_chat_worker.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include "openvino/genai/llm_pipeline.hpp" + +using namespace Napi; + +class StartChatWorker : public AsyncWorker { + public: + StartChatWorker(Function& callback, std::shared_ptr& pipe); + virtual ~StartChatWorker(){}; + + void Execute(); + void OnOK(); + + private: + std::shared_ptr& pipe; +}; diff --git a/src/js/lib/bindings.cjs b/src/js/lib/bindings.cjs new file mode 100644 index 0000000000..acd9e590b8 --- /dev/null +++ b/src/js/lib/bindings.cjs @@ -0,0 +1 @@ +module.exports = require('../bin/genai_node_addon.node'); diff --git a/src/js/lib/module.js b/src/js/lib/module.js new file mode 100644 index 0000000000..2f3d557711 --- /dev/null +++ b/src/js/lib/module.js @@ -0,0 +1,118 @@ +import util from 'node:util'; + +import addon from './bindings.cjs'; + +class LLMPipeline { + modelPath = null; + device = null; + pipeline = null; + isInitialized = false; + isChatStarted = false; + + constructor(modelPath, device) { + this.modelPath = modelPath; + this.device = device; + } + + async init() { + if (this.isInitialized) + throw new Error('Pipeline is already initialized'); + + this.pipeline = new addon.LLMPipeline(); + + const init = util.promisify(this.pipeline.init.bind(this.pipeline)); + const result = await init(this.modelPath, this.device); + + this.isInitialized = true; + + return result; + } + + async startChat() { + if (this.isChatStarted) + throw new Error('Chat is already started'); + + const startChatPromise = util.promisify( + this.pipeline.startChat.bind(this.pipeline) + ); + const result = await startChatPromise(); + + this.isChatStarted = true; + + return result; + } + async finishChat() { + if (!this.isChatStarted) + throw new Error('Chat is not started'); + + const finishChatPromise = util.promisify( + this.pipeline.finishChat.bind(this.pipeline) + ); + const result = await finishChatPromise(); + + this.isChatStarted = false; + + return result; + } + + async generate(prompt, generationCallback, options = {}) { + if (!this.isInitialized) + throw new Error('Pipeline is not initialized'); + + if (typeof prompt !== 'string') + throw new Error('Prompt must be a string'); + if (typeof generationCallback !== 'function') + throw new Error('Generation callback must be a function'); + if (typeof options !== 'object') + throw new Error('Options must be an object'); + + let result = ''; + const castedOptions = {}; + + for (const key in options) castedOptions[key] = String(options[key]); + + const promise = new Promise((resolve, reject) => { + const generationCallbackDecorator = function(isDone, chunk) { + if (isDone) return resolve(result); + + result += chunk; + + try { + generationCallback(chunk); + } catch (err) { + reject(err); + } + }; + + try { + this.pipeline.generate(prompt, generationCallbackDecorator, castedOptions); + } catch (err) { + reject(err); + } + }); + + return promise; + } +} + +const availablePipelines = { LLMPipeline: LLMPipeline }; + +class Pipeline { + static async create(pipelineType, modelPath, device = 'CPU') { + if (!Object.keys(availablePipelines).includes(pipelineType)) + throw new Error(`Pipeline type: '${pipelineType}' doesn't support`); + + const pipeline = new availablePipelines[pipelineType](modelPath, device); + await pipeline.init(); + + return pipeline; + } +} + +const availablePipelinesKeys = Object.keys(availablePipelines); + +export { + addon, + Pipeline, + availablePipelinesKeys as availablePipelines, +}; diff --git a/src/js/package-lock.json b/src/js/package-lock.json new file mode 100644 index 0000000000..5964072048 --- /dev/null +++ b/src/js/package-lock.json @@ -0,0 +1,21 @@ +{ + "name": "genai-node", + "version": "2024.5.0-preview", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "genai-node", + "version": "2024.5.0-preview", + "license": "Apache-2.0", + "os": [ + "linux", + "darwin", + "win32" + ], + "engines": { + "node": ">=21.0.0" + } + } + } +} diff --git a/src/js/package.json b/src/js/package.json new file mode 100644 index 0000000000..89335285cb --- /dev/null +++ b/src/js/package.json @@ -0,0 +1,24 @@ +{ + "name": "genai-node", + "type": "module", + "version": "2024.5.0-preview", + "description": "OpenVINO™ GenAI pipelines for using from Node.js environment", + "license": "Apache-2.0", + "main": "./lib/module.js", + "os": [ + "linux", + "darwin", + "win32" + ], + "engines": { + "node": ">=21.0.0" + }, + "keywords": [ + "OpenVINO", + "OpenVINO GenAI", + "GenAI" + ], + "scripts": { + "test": "node --test ./tests/*.test.js" + } +} diff --git a/src/js/src/addon.cpp b/src/js/src/addon.cpp new file mode 100644 index 0000000000..4bd1da7bb6 --- /dev/null +++ b/src/js/src/addon.cpp @@ -0,0 +1,30 @@ +#include +#include + +#include "include/addon.hpp" + +#include "include/llm_pipeline/llm_pipeline_wrapper.hpp" + +void init_class(Napi::Env env, + Napi::Object exports, + std::string class_name, + Prototype func, + Napi::FunctionReference& reference) { + const auto& prototype = func(env); + + reference = Napi::Persistent(prototype); + exports.Set(class_name, prototype); +} + +// Define the addon initialization function +Napi::Object init_module(Napi::Env env, Napi::Object exports) { + auto addon_data = new AddonData(); + env.SetInstanceData(addon_data); + + init_class(env, exports, "LLMPipeline", &LLMPipelineWrapper::get_class, addon_data->core); + + return exports; +} + +// Register the addon with Node.js +NODE_API_MODULE(genai-node, init_module) diff --git a/src/js/src/helper.cpp b/src/js/src/helper.cpp new file mode 100644 index 0000000000..106994603b --- /dev/null +++ b/src/js/src/helper.cpp @@ -0,0 +1,53 @@ +#include "include/helper.hpp" + +ov::AnyMap to_anyMap(const Napi::Env& env, const Napi::Value& val) { + ov::AnyMap properties; + if (!val.IsObject()) { + OPENVINO_THROW("Passed Napi::Value must be an object."); + } + const auto& parameters = val.ToObject(); + const auto& keys = parameters.GetPropertyNames(); + + for (uint32_t i = 0; i < keys.Length(); ++i) { + const auto& property_name = static_cast(keys[i]).ToString().Utf8Value(); + + const auto& any_value = js_to_cpp(env, parameters.Get(property_name)); + + properties.insert(std::make_pair(property_name, any_value)); + } + + return properties; +} + +template <> +ov::Any js_to_cpp(const Napi::Env& env, const Napi::Value& value) { + if (value.IsString()) { + return ov::Any(value.ToString().Utf8Value()); + } else if (value.IsBigInt()) { + Napi::BigInt big_value = value.As(); + bool is_lossless; + int64_t big_num = big_value.Int64Value(&is_lossless); + + if (!is_lossless) { + OPENVINO_THROW("Result of BigInt conversion to int64_t results in a loss of precision"); + } + + return ov::Any(big_num); + } else if (value.IsNumber()) { + Napi::Number num = value.ToNumber(); + + if (is_napi_value_int(env, value)) { + return ov::Any(num.Int32Value()); + } else { + return ov::Any(num.DoubleValue()); + } + } else if (value.IsBoolean()) { + return ov::Any(value.ToBoolean()); + } else { + OPENVINO_THROW("Cannot convert to ov::Any"); + } +} + +bool is_napi_value_int(const Napi::Env& env, const Napi::Value& num) { + return env.Global().Get("Number").ToObject().Get("isInteger").As().Call({num}).ToBoolean().Value(); +} diff --git a/src/js/src/llm_pipeline/finish_chat_worker.cpp b/src/js/src/llm_pipeline/finish_chat_worker.cpp new file mode 100644 index 0000000000..b07284688c --- /dev/null +++ b/src/js/src/llm_pipeline/finish_chat_worker.cpp @@ -0,0 +1,14 @@ +#include "include/llm_pipeline/finish_chat_worker.hpp" +#include +#include + +FinishChatWorker::FinishChatWorker(Function& callback, std::shared_ptr& pipe) + : AsyncWorker(callback), pipe(pipe) {}; + +void FinishChatWorker::Execute() { + this->pipe->finish_chat(); +}; + +void FinishChatWorker::OnOK() { + Callback().Call({ Env().Null() }); +}; diff --git a/src/js/src/llm_pipeline/init_worker.cpp b/src/js/src/llm_pipeline/init_worker.cpp new file mode 100644 index 0000000000..87dd1aaf34 --- /dev/null +++ b/src/js/src/llm_pipeline/init_worker.cpp @@ -0,0 +1,18 @@ +#include "include/llm_pipeline/init_worker.hpp" +#include +#include + +InitWorker::InitWorker( + Function& callback, + std::shared_ptr& pipe, + const std::string model_path, + const std::string device +) : AsyncWorker(callback), pipe(pipe), model_path(model_path), device(device) {}; + +void InitWorker::Execute() { + this->pipe = std::make_shared(this->model_path, this->device); +}; + +void InitWorker::OnOK() { + Callback().Call({ Env().Null() }); +}; diff --git a/src/js/src/llm_pipeline/llm_pipeline_wrapper.cpp b/src/js/src/llm_pipeline/llm_pipeline_wrapper.cpp new file mode 100644 index 0000000000..df1cfceadc --- /dev/null +++ b/src/js/src/llm_pipeline/llm_pipeline_wrapper.cpp @@ -0,0 +1,151 @@ +#include "include/helper.hpp" + +#include "include/llm_pipeline/llm_pipeline_wrapper.hpp" +#include "include/llm_pipeline/start_chat_worker.hpp" +#include "include/llm_pipeline/finish_chat_worker.hpp" +#include "include/llm_pipeline/init_worker.hpp" + +struct TsfnContext { + TsfnContext(std::string prompt) : prompt(prompt) {}; + ~TsfnContext() { + // std::cout << "Tsfn destructed" << std::endl; + }; + + std::thread native_thread; + Napi::ThreadSafeFunction tsfn; + + std::string prompt; + std::shared_ptr pipe = nullptr; + std::shared_ptr options = nullptr; +}; + +void performInferenceThread(TsfnContext* context) { + auto callback = [](Napi::Env env, Napi::Function js_callback, TsfnContext* context) { + try { + std::function streamer = [env, js_callback](std::string word) { + js_callback.Call({ + Napi::Boolean::New(env, false), + Napi::String::New(env, word) + }); + + // Return flag corresponds whether generation should be stopped. + // false means continue generation. + return false; + }; + + ov::genai::GenerationConfig config; + + config.update_generation_config(*context->options); + + context->pipe->generate(context->prompt, config, streamer); + js_callback.Call({ + Napi::Boolean::New(env, true) + }); + } catch(std::exception& err) { + Napi::Error::Fatal("performInferenceThread callback error. Details:" , err.what()); + } + }; + + try { + napi_status status = context->tsfn.BlockingCall(context, callback); + if (status != napi_ok) { + // Handle error + Napi::Error::Fatal("performInferenceThread error", "napi_status != napi_ok"); + } + + context->tsfn.Release(); + } + catch(std::exception& e) { + Napi::Error::Fatal("performInferenceThread error" , e.what()); + + context->tsfn.Release(); + } +} + +LLMPipelineWrapper::LLMPipelineWrapper(const Napi::CallbackInfo& info) : Napi::ObjectWrap(info) {}; + +Napi::Function LLMPipelineWrapper::get_class(Napi::Env env) { + return DefineClass(env, + "LLMPipeline", + {InstanceMethod("init", &LLMPipelineWrapper::init), + InstanceMethod("generate", &LLMPipelineWrapper::generate), + InstanceMethod("startChat", &LLMPipelineWrapper::start_chat), + InstanceMethod("finishChat", &LLMPipelineWrapper::finish_chat)}); +} + +Napi::Value LLMPipelineWrapper::init(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + const std::string model_path = info[0].ToString(); + const std::string device = info[1].ToString(); + Napi::Function callback = info[2].As(); + + InitWorker* asyncWorker = new InitWorker(callback, this->pipe, model_path, device); + asyncWorker->Queue(); + + return info.Env().Undefined(); +} + +Napi::Value LLMPipelineWrapper::generate(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + TsfnContext* context = nullptr; + + try { + std::string prompt = info[0].ToString(); + ov::AnyMap options; + if (info.Length() == 3) { + options = to_anyMap(info.Env(), info[2]); + } + + context = new TsfnContext(prompt); + context->pipe = this->pipe; + context->options = std::make_shared(options); + // Create a ThreadSafeFunction + context->tsfn = Napi::ThreadSafeFunction::New( + env, + info[1].As(), // JavaScript function called asynchronously + "TSFN", // Name + 0, // Unlimited queue + 1, // Only one thread will use this initially + [context](Napi::Env) { // Finalizer used to clean threads up + // std::cout << "Finalize TFSN" << std::endl; + context->native_thread.join(); + delete context; + } + ); + context->native_thread = std::thread(performInferenceThread, context); + + return Napi::Boolean::New(env, false); + } catch(Napi::TypeError& type_err) { + throw type_err; + } catch(std::exception& err) { + std::cout << "Catch in the thread: '" << err.what() << "'" << std::endl; + if (context != nullptr) { + context->tsfn.Release(); + } + + throw Napi::Error::New(env, err.what()); + } + + return Napi::Boolean::New(env, true); +} + +Napi::Value LLMPipelineWrapper::start_chat(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + Napi::Function callback = info[0].As(); + + StartChatWorker* asyncWorker = new StartChatWorker(callback, this->pipe); + asyncWorker->Queue(); + + return info.Env().Undefined(); +} + +Napi::Value LLMPipelineWrapper::finish_chat(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + + Napi::Function callback = info[0].As(); + + FinishChatWorker* asyncWorker = new FinishChatWorker(callback, this->pipe); + asyncWorker->Queue(); + + return info.Env().Undefined(); +} diff --git a/src/js/src/llm_pipeline/start_chat_worker.cpp b/src/js/src/llm_pipeline/start_chat_worker.cpp new file mode 100644 index 0000000000..302c505105 --- /dev/null +++ b/src/js/src/llm_pipeline/start_chat_worker.cpp @@ -0,0 +1,14 @@ +#include "include/llm_pipeline/start_chat_worker.hpp" +#include +#include + +StartChatWorker::StartChatWorker(Function& callback, std::shared_ptr& pipe) + : AsyncWorker(callback), pipe(pipe) {}; + +void StartChatWorker::Execute() { + this->pipe->start_chat(); +}; + +void StartChatWorker::OnOK() { + Callback().Call({ Env().Null() }); +}; diff --git a/src/js/tests/bindings.test.js b/src/js/tests/bindings.test.js new file mode 100644 index 0000000000..2b031124f0 --- /dev/null +++ b/src/js/tests/bindings.test.js @@ -0,0 +1,71 @@ +import addon from '../lib/bindings.cjs'; + +import assert from 'node:assert'; +import { describe, it, before, after } from 'node:test'; + +const MODEL_PATH = process.env.MODEL_PATH; + +describe('bindings', () => { + let pipeline = null; + + before((_, done) => { + pipeline = new addon.LLMPipeline(); + + pipeline.init(MODEL_PATH, 'AUTO', (err) => { + if (err) { + console.error(err); + process.exit(1); + } + + pipeline.startChat((err) => { + if (err) { + console.error(err); + process.exit(1); + } + + done(); + }); + }); + }); + + after((_, done) => { + pipeline.finishChat((err) => { + if (err) { + console.error(err); + process.exit(1); + } + + done(); + }); + }); + + it('should generate string result', (_, done) => { + let output = ''; + + pipeline.generate('Say Hello', (isDone, chunk) => { + if (!isDone) { + output += chunk; + + return; + } + }, { temperature: '0', max_new_tokens: '4' }); + + assert.ok(true); + done(); + }); + + // it('should generate "Hello world"', (_, done) => { + // let output = ''; + + // pipeline.generate('Type "Hello world!" in English', (isDone, chunk) => { + // if (!isDone) { + // output += chunk; + + // return; + // } + + // assert.strictEqual(output, '"Hello world!"'); + // done(); + // }, { temperature: '0', max_new_tokens: '4' }); + // }); +}); diff --git a/src/js/tests/module.test.js b/src/js/tests/module.test.js new file mode 100644 index 0000000000..167cf91436 --- /dev/null +++ b/src/js/tests/module.test.js @@ -0,0 +1,140 @@ +import { Pipeline } from '../lib/module.js'; + +import assert from 'node:assert/strict'; +import { describe, it, before, after } from 'node:test'; + +const MODEL_PATH = process.env.MODEL_PATH; + +// describe('module', async () => { +// let pipeline = null; + +// await before(async () => { +// pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + +// await pipeline.startChat(); +// }); + +// await after(async () => { +// await pipeline.finishChat(); +// }); + +// await it('should generate "Hello world"', async () => { +// const result = await pipeline.generate( +// 'Type "Hello world!" in English', +// () => {}, +// { temperature: '0', max_new_tokens: '4' } +// ); + +// assert.strictEqual(result, '"Hello world!"'); +// }); +// }); + +describe('corner cases', async () => { + it('should throw an error if pipeline is already initialized', async () => { + const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + + await assert.rejects( + async () => await pipeline.init(), + { + name: 'Error', + message: 'Pipeline is already initialized', + }, + ); + }); + + it('should throw an error if chat is already started', async () => { + const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + + await pipeline.startChat(); + + await assert.rejects( + () => pipeline.startChat(), + { + name: 'Error', + message: 'Chat is already started', + }, + ); + }); + + it('should throw an error if chat is not started', async () => { + const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + + await assert.rejects( + () => pipeline.finishChat(), + { + name: 'Error', + message: 'Chat is not started', + }, + ); + }); +}); + +describe('generation parameters validation', () => { + let pipeline = null; + + before(async () => { + pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + + await pipeline.startChat(); + }); + + after(async () => { + await pipeline.finishChat(); + }); + + it('should throw an error if temperature is not a number', async () => { + await assert.rejects( + async () => await pipeline.generate(), + { + name: 'Error', + message: 'Prompt must be a string', + }, + ); + }); + + it('should throw an error if generationCallback is not a function', async () => { + const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + + await pipeline.startChat(); + + await assert.rejects( + async () => await pipeline.generate('prompt'), + { + name: 'Error', + message: 'Generation callback must be a function', + }, + ); + }); + + it('should throw an error if options specified but not an object', async () => { + await assert.rejects( + async () => await pipeline.generate('prompt', () => {}, 'options'), + { + name: 'Error', + message: 'Options must be an object', + }, + ); + }); + + it('should perform generation with default options', async () => { + try { + await pipeline.generate('prompt', () => {}, { max_new_tokens: 1 }); + } catch (error) { + assert.fail(error); + } + + assert.ok(true); + }); + + it('should return a string as generation result', async () => { + const reply = await pipeline.generate('prompt', () => {}, { max_new_tokens: 1 }); + + assert.strictEqual(typeof reply, 'string'); + }); + + it('should call generationCallback with string chunk', async () => { + await pipeline.generate('prompt', (chunk) => { + assert.strictEqual(typeof chunk, 'string'); + }, { max_new_tokens: 1 }); + }); +}); diff --git a/src/js/tests/setup.js b/src/js/tests/setup.js new file mode 100644 index 0000000000..b39f7c0136 --- /dev/null +++ b/src/js/tests/setup.js @@ -0,0 +1,9 @@ +import { testModels, downloadTestModel } from './utils.js'; + +if (require.main === module) { + main(); +} + +async function main() { + await downloadTestModel(testModels.testModelFP32); +} diff --git a/src/js/tests/utils.js b/src/js/tests/utils.js new file mode 100644 index 0000000000..c2089e30b4 --- /dev/null +++ b/src/js/tests/utils.js @@ -0,0 +1,73 @@ +// -*- coding: utf-8 -*- +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +const path = require('path'); +const fs = require('node:fs/promises'); +const { + downloadFile, + checkIfPathExists, +} = require('../../scripts/download_runtime'); + +const modelDir = 'tests/unit/test_models/'; +const testModels = { + testModelFP32: { + xml: 'test_model_fp32.xml', + bin: 'test_model_fp32.bin', + xmlURL: + 'https://raw.githubusercontent.com/openvinotoolkit/testdata/master/models/test_model/test_model_fp32.xml', + binURL: + 'https://media.githubusercontent.com/media/openvinotoolkit/testdata/master/models/test_model/test_model_fp32.bin', + }, +}; + +module.exports = { + getModelPath, + downloadTestModel, + isModelAvailable, + testModels, +}; + +function getModelPath(isFP16 = false) { + const modelName = `test_model_fp${isFP16 ? 16 : 32}`; + + return { + xml: path.join(modelDir, `${modelName}.xml`), + bin: path.join(modelDir, `${modelName}.bin`), + }; +} + +async function downloadTestModel(model) { + const modelsDir = './tests/unit/test_models'; + try { + const ifModelsDirectoryExists = await checkIfPathExists(modelsDir); + if (!ifModelsDirectoryExists) { + await fs.mkdir(modelDir); + } + + const modelPath = path.join(modelsDir, model.xml); + const modelExists = await checkIfPathExists(modelPath); + if (modelExists) return; + + const { env } = process; + const proxyUrl = env.http_proxy || env.HTTP_PROXY || env.npm_config_proxy; + + await downloadFile(model.xmlURL, modelsDir, model.xml, proxyUrl); + await downloadFile(model.binURL, modelsDir, model.bin, proxyUrl); + } catch(error) { + console.error(`Failed to download the model: ${error}.`); + throw error; + } +} + +async function isModelAvailable(model) { + const baseArtifactsDir = './tests/unit/test_models'; + const modelPath = path.join(baseArtifactsDir, model.xml); + const modelExists = await checkIfPathExists(modelPath); + if (modelExists) return; + + console.log( + '\n\nTestModel cannot be found.\nPlease run `npm run test_setup`.\n\n', + ); + process.exit(1); +} From 2d71b6b4ac14191dbd5330305aae22edee7f9962 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 13 Nov 2024 18:36:50 +0100 Subject: [PATCH 02/47] Include js bindings into package build --- src/js/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/js/CMakeLists.txt b/src/js/CMakeLists.txt index 96ba0019c1..d7ea811618 100644 --- a/src/js/CMakeLists.txt +++ b/src/js/CMakeLists.txt @@ -87,6 +87,6 @@ if(rpaths) endif() install(TARGETS ${PROJECT_NAME} - LIBRARY DESTINATION openvino_genai COMPONENT ${PROJECT_NAME} EXCLUDE_FROM_ALL - RUNTIME DESTINATION openvino_genai COMPONENT ${PROJECT_NAME} EXCLUDE_FROM_ALL + LIBRARY DESTINATION . COMPONENT ${PROJECT_NAME} + RUNTIME DESTINATION . COMPONENT ${PROJECT_NAME} ) From 0c0b41ca817b9326f493f9fcb285c7e7237a260c Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Fri, 15 Nov 2024 15:48:07 +0100 Subject: [PATCH 03/47] Put binaries at the top level of NPM package --- src/cpp/CMakeLists.txt | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt index 49b640763d..f689547a69 100644 --- a/src/cpp/CMakeLists.txt +++ b/src/cpp/CMakeLists.txt @@ -118,13 +118,29 @@ if(MSVC OR APPLE) set(ARCH_DIR ${ARCH_DIR}/${CMAKE_BUILD_TYPE}) endif() +# Put binaries at the top level for NPM package +if(CPACK_GENERATOR STREQUAL "NPM") + set(LIBRARY_DESTINATION .) + set(ARCHIVE_DESTINATION .) + set(RUNTIME_DESTINATION .) +else() + set(LIBRARY_DESTINATION runtime/lib/${ARCH_DIR}) + set(ARCHIVE_DESTINATION runtime/lib/${ARCH_DIR}) + set(RUNTIME_DESTINATION runtime/bin/${ARCH_DIR}) +endif() + install(TARGETS ${TARGET_NAME} EXPORT OpenVINOGenAITargets - LIBRARY DESTINATION runtime/lib/${ARCH_DIR} COMPONENT core_genai + LIBRARY DESTINATION ${LIBRARY_DESTINATION} COMPONENT core_genai NAMELINK_COMPONENT core_genai_dev - ARCHIVE DESTINATION runtime/lib/${ARCH_DIR} COMPONENT core_genai_dev - RUNTIME DESTINATION runtime/bin/${ARCH_DIR} COMPONENT core_genai + ARCHIVE DESTINATION ${ARCHIVE_DESTINATION} COMPONENT core_genai_dev + RUNTIME DESTINATION ${RUNTIME_DESTINATION} COMPONENT core_genai INCLUDES DESTINATION runtime/include) +# samples do not need to be built for NPM package +if(CPACK_GENERATOR STREQUAL "NPM") + return() +endif() + install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION runtime/include COMPONENT core_genai_dev) install(EXPORT OpenVINOGenAITargets FILE OpenVINOGenAITargets.cmake @@ -133,9 +149,9 @@ install(EXPORT OpenVINOGenAITargets FILE OpenVINOGenAITargets.cmake include(CMakePackageConfigHelpers) configure_package_config_file("${OpenVINOGenAI_SOURCE_DIR}/cmake/templates/OpenVINOGenAIConfig.cmake.in" - "${CMAKE_BINARY_DIR}/OpenVINOGenAIConfig.cmake" INSTALL_DESTINATION runtime/cmake) + "${CMAKE_BINARY_DIR}/OpenVINOGenAIConfig.cmake" INSTALL_DESTINATION runtime/cmake) write_basic_package_version_file("${CMAKE_BINARY_DIR}/OpenVINOGenAIConfigVersion.cmake" - VERSION ${OpenVINOGenAI_VERSION} COMPATIBILITY AnyNewerVersion) + VERSION ${OpenVINOGenAI_VERSION} COMPATIBILITY AnyNewerVersion) install(FILES "${CMAKE_BINARY_DIR}/OpenVINOGenAIConfig.cmake" "${CMAKE_BINARY_DIR}/OpenVINOGenAIConfigVersion.cmake" DESTINATION runtime/cmake COMPONENT core_genai_dev) export(EXPORT OpenVINOGenAITargets FILE "${CMAKE_BINARY_DIR}/OpenVINOGenAITargets.cmake" NAMESPACE openvino::) From 7e80306dacb46ba8ea79fdd455c057e926a52421 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Fri, 15 Nov 2024 15:48:58 +0100 Subject: [PATCH 04/47] Skip samples for NPM package --- samples/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 229eccb3fe..dbb9639128 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -2,6 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 # +# Samples do not need to be built for NPM package +if(CPACK_GENERATOR STREQUAL "NPM") + return() +endif() + add_subdirectory(cpp/beam_search_causal_lm) add_subdirectory(cpp/benchmark_genai) add_subdirectory(cpp/chat_sample) From 67d5a404c2b90ceb82b2f98979a9a189e8d9b23e Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 19 Nov 2024 17:56:37 +0100 Subject: [PATCH 05/47] Add model downloading into js unit tests --- src/js/.gitignore | 2 + src/js/package-lock.json | 449 ++++++++++++++++++++++++++++++++++ src/js/package.json | 8 +- src/js/tests/bindings.test.js | 26 +- src/js/tests/models.js | 3 + src/js/tests/module.test.js | 40 +-- src/js/tests/setup.js | 11 +- src/js/tests/utils.js | 96 +++----- 8 files changed, 535 insertions(+), 100 deletions(-) create mode 100644 src/js/tests/models.js diff --git a/src/js/.gitignore b/src/js/.gitignore index e7fffbc37a..638d8a5bb7 100644 --- a/src/js/.gitignore +++ b/src/js/.gitignore @@ -3,3 +3,5 @@ bin bin.* build thirdparty +node_modules +tests/models diff --git a/src/js/package-lock.json b/src/js/package-lock.json index 5964072048..4da5b57ea7 100644 --- a/src/js/package-lock.json +++ b/src/js/package-lock.json @@ -13,9 +13,458 @@ "darwin", "win32" ], + "devDependencies": { + "@huggingface/hub": "^0.21.0", + "global-agent": "^3.0.0", + "node-fetch": "^3.3.2" + }, "engines": { "node": ">=21.0.0" } + }, + "node_modules/@huggingface/hub": { + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/@huggingface/hub/-/hub-0.21.0.tgz", + "integrity": "sha512-DpitNhqobMJLTv8dUq/EMtrz1dpfs3UrSVCxe1aKpjLAdOs6Gm6rqrinUFNvC9G88RIRzIYzojUtYUqlkKwKnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@huggingface/tasks": "^0.13.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@huggingface/tasks": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.13.4.tgz", + "integrity": "sha512-LETHbMSK3gHBFU0D09ziEJm6t1Pcgii4SFwHw+d+8MFGfkAryxaDl2qaHY+PxiTkZEeaTLd6G8/239SJuVxyWg==", + "dev": true, + "license": "MIT" + }, + "node_modules/boolean": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", + "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "dev": true, + "license": "MIT" + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/define-properties": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", + "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", + "dev": true, + "license": "MIT", + "dependencies": { + "define-data-property": "^1.0.1", + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "dev": true, + "license": "MIT" + }, + "node_modules/es-define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", + "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-intrinsic": "^1.2.4" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", + "dev": true, + "license": "MIT" + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", + "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/global-agent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", + "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "es6-error": "^4.1.1", + "matcher": "^3.0.0", + "roarr": "^2.15.3", + "semver": "^7.3.2", + "serialize-error": "^7.0.1" + }, + "engines": { + "node": ">=10.0" + } + }, + "node_modules/globalthis": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", + "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "define-properties": "^1.2.1", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gopd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", + "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-intrinsic": "^1.1.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-proto": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", + "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", + "dev": true, + "license": "ISC" + }, + "node_modules/matcher": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", + "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", + "dev": true, + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "dev": true, + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/roarr": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", + "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "detect-node": "^2.0.4", + "globalthis": "^1.0.1", + "json-stringify-safe": "^5.0.1", + "semver-compare": "^1.0.0", + "sprintf-js": "^1.1.2" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/semver-compare": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", + "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", + "dev": true, + "license": "MIT" + }, + "node_modules/serialize-error": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", + "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", + "dev": true, + "license": "MIT", + "dependencies": { + "type-fest": "^0.13.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/type-fest": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } } } } diff --git a/src/js/package.json b/src/js/package.json index 89335285cb..5b069fb01f 100644 --- a/src/js/package.json +++ b/src/js/package.json @@ -19,6 +19,12 @@ "GenAI" ], "scripts": { - "test": "node --test ./tests/*.test.js" + "test_setup": "node ./tests/setup.js", + "test": "npm run test_setup && node --test ./tests/*.test.js" + }, + "devDependencies": { + "node-fetch": "^3.3.2", + "global-agent": "^3.0.0", + "@huggingface/hub": "^0.21.0" } } diff --git a/src/js/tests/bindings.test.js b/src/js/tests/bindings.test.js index 2b031124f0..3e852e5307 100644 --- a/src/js/tests/bindings.test.js +++ b/src/js/tests/bindings.test.js @@ -2,8 +2,10 @@ import addon from '../lib/bindings.cjs'; import assert from 'node:assert'; import { describe, it, before, after } from 'node:test'; +import { models } from './models.js'; -const MODEL_PATH = process.env.MODEL_PATH; +const MODEL_PATH = process.env.MODEL_PATH + || `./tests/models/${models[0].split('/')[1]}`; describe('bindings', () => { let pipeline = null; @@ -54,18 +56,18 @@ describe('bindings', () => { done(); }); - // it('should generate "Hello world"', (_, done) => { - // let output = ''; + it('should generate "Hello world"', (_, done) => { + let output = ''; - // pipeline.generate('Type "Hello world!" in English', (isDone, chunk) => { - // if (!isDone) { - // output += chunk; + pipeline.generate('Type "Hello world!" in English', (isDone, chunk) => { + if (!isDone) { + output += chunk; - // return; - // } + return; + } - // assert.strictEqual(output, '"Hello world!"'); - // done(); - // }, { temperature: '0', max_new_tokens: '4' }); - // }); + assert.strictEqual(output, 'Hello world!'); + done(); + }, { temperature: '0', max_new_tokens: '4' }); + }); }); diff --git a/src/js/tests/models.js b/src/js/tests/models.js new file mode 100644 index 0000000000..b7f7505464 --- /dev/null +++ b/src/js/tests/models.js @@ -0,0 +1,3 @@ +export const models = [ + 'AIFunOver/Llama-3.2-3B-Instruct-openvino-8bit', +]; diff --git a/src/js/tests/module.test.js b/src/js/tests/module.test.js index 167cf91436..90b2f5266d 100644 --- a/src/js/tests/module.test.js +++ b/src/js/tests/module.test.js @@ -2,32 +2,34 @@ import { Pipeline } from '../lib/module.js'; import assert from 'node:assert/strict'; import { describe, it, before, after } from 'node:test'; +import { models } from './models.js'; -const MODEL_PATH = process.env.MODEL_PATH; +const MODEL_PATH = process.env.MODEL_PATH + || `./tests/models/${models[0].split('/')[1]}`; -// describe('module', async () => { -// let pipeline = null; +describe('module', async () => { + let pipeline = null; -// await before(async () => { -// pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + await before(async () => { + pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); -// await pipeline.startChat(); -// }); + await pipeline.startChat(); + }); -// await after(async () => { -// await pipeline.finishChat(); -// }); + await after(async () => { + await pipeline.finishChat(); + }); -// await it('should generate "Hello world"', async () => { -// const result = await pipeline.generate( -// 'Type "Hello world!" in English', -// () => {}, -// { temperature: '0', max_new_tokens: '4' } -// ); + await it('should generate "Hello world"', async () => { + const result = await pipeline.generate( + 'Type "Hello world!" in English', + () => {}, + { temperature: '0', max_new_tokens: '4' } + ); -// assert.strictEqual(result, '"Hello world!"'); -// }); -// }); + assert.strictEqual(result, 'Hello world!'); + }); +}); describe('corner cases', async () => { it('should throw an error if pipeline is already initialized', async () => { diff --git a/src/js/tests/setup.js b/src/js/tests/setup.js index b39f7c0136..3b52651719 100644 --- a/src/js/tests/setup.js +++ b/src/js/tests/setup.js @@ -1,9 +1,6 @@ -import { testModels, downloadTestModel } from './utils.js'; +import { dowloadModel } from './utils.js'; +import { models } from './models.js'; -if (require.main === module) { - main(); -} - -async function main() { - await downloadTestModel(testModels.testModelFP32); +for (const model of models) { + await dowloadModel(model); } diff --git a/src/js/tests/utils.js b/src/js/tests/utils.js index c2089e30b4..504782d8d1 100644 --- a/src/js/tests/utils.js +++ b/src/js/tests/utils.js @@ -1,73 +1,47 @@ -// -*- coding: utf-8 -*- -// Copyright (C) 2018-2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 +import { bootstrap } from 'global-agent'; +import { promises as fs } from 'node:fs'; +import { listFiles, downloadFile } from '@huggingface/hub'; -const path = require('path'); -const fs = require('node:fs/promises'); -const { - downloadFile, - checkIfPathExists, -} = require('../../scripts/download_runtime'); +const BASE_DIR = './tests/models/'; -const modelDir = 'tests/unit/test_models/'; -const testModels = { - testModelFP32: { - xml: 'test_model_fp32.xml', - bin: 'test_model_fp32.bin', - xmlURL: - 'https://raw.githubusercontent.com/openvinotoolkit/testdata/master/models/test_model/test_model_fp32.xml', - binURL: - 'https://media.githubusercontent.com/media/openvinotoolkit/testdata/master/models/test_model/test_model_fp32.bin', - }, -}; +bootstrap(); -module.exports = { - getModelPath, - downloadTestModel, - isModelAvailable, - testModels, -}; +export async function dowloadModel(repo) { + console.log(`Downloading model '${repo}'`); -function getModelPath(isFP16 = false) { - const modelName = `test_model_fp${isFP16 ? 16 : 32}`; + const fetch = await import('node-fetch'); + const modelName = repo.split('/')[1]; + const destDir = `${BASE_DIR}${modelName}`; - return { - xml: path.join(modelDir, `${modelName}.xml`), - bin: path.join(modelDir, `${modelName}.bin`), - }; -} - -async function downloadTestModel(model) { - const modelsDir = './tests/unit/test_models'; - try { - const ifModelsDirectoryExists = await checkIfPathExists(modelsDir); - if (!ifModelsDirectoryExists) { - await fs.mkdir(modelDir); - } - - const modelPath = path.join(modelsDir, model.xml); - const modelExists = await checkIfPathExists(modelPath); - if (modelExists) return; + await fs.mkdir(destDir, { recursive: true }); - const { env } = process; - const proxyUrl = env.http_proxy || env.HTTP_PROXY || env.npm_config_proxy; + const fileList = await listFiles({ + repo, + fetch: fetch.default, + }); + const fileNames = []; + for await (const file of fileList) { + fileNames.push(file.path); + } - await downloadFile(model.xmlURL, modelsDir, model.xml, proxyUrl); - await downloadFile(model.binURL, modelsDir, model.bin, proxyUrl); - } catch(error) { - console.error(`Failed to download the model: ${error}.`); - throw error; + for (const path of fileNames) { + console.log(`Downloading file '${path}'`); + const response = await downloadFile({ + repo, + path, + fetch: fetch.default, + }); + const filename = `${destDir}/${path}`; + + await saveFile(filename, response); + console.log(`File '${path}' downloaded`); } + + console.log(`Model '${repo}' downloaded`); } -async function isModelAvailable(model) { - const baseArtifactsDir = './tests/unit/test_models'; - const modelPath = path.join(baseArtifactsDir, model.xml); - const modelExists = await checkIfPathExists(modelPath); - if (modelExists) return; +async function saveFile(file, response) { + const arrayBuffer = await response.arrayBuffer(); - console.log( - '\n\nTestModel cannot be found.\nPlease run `npm run test_setup`.\n\n', - ); - process.exit(1); + await fs.writeFile(file, Buffer.from(arrayBuffer)); } From 685344611228f11dca98194c11ae5eae10fcef87 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 20 Nov 2024 15:01:14 +0100 Subject: [PATCH 06/47] Set openvino_tokenizers on latest master --- thirdparty/openvino_tokenizers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thirdparty/openvino_tokenizers b/thirdparty/openvino_tokenizers index 306dcd8dae..aeb1bc2cb1 160000 --- a/thirdparty/openvino_tokenizers +++ b/thirdparty/openvino_tokenizers @@ -1 +1 @@ -Subproject commit 306dcd8daec36bbc680c50c68de1e954f42b0ab8 +Subproject commit aeb1bc2cb1646650d6ff0fa258ca775bd20796d6 From ae806da5d252e90d1fc72076194bac1dbe59e7e7 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 26 Nov 2024 15:55:04 +0100 Subject: [PATCH 07/47] Temporary disable most of gha wf to debug --- .github/workflows/assign_issue.yml | 10 ++++----- .github/workflows/bandit.yml | 10 ++++----- .github/workflows/causal_lm_cpp.yml | 16 +++++++------- .github/workflows/job_vlm_sample_llava.yml | 18 +++++++-------- .github/workflows/labeler.yml | 4 ++-- .github/workflows/lcm_dreamshaper_cpp.yml | 22 +++++++++---------- .github/workflows/llm_bench-python.yml | 22 +++++++++---------- .github/workflows/mac.yml | 16 +++++++------- .../workflows/stable_diffusion_1_5_cpp.yml | 14 ++++++------ .github/workflows/windows.yml | 16 +++++++------- 10 files changed, 74 insertions(+), 74 deletions(-) diff --git a/.github/workflows/assign_issue.yml b/.github/workflows/assign_issue.yml index 4a4579e2c7..68bfb6cf2d 100644 --- a/.github/workflows/assign_issue.yml +++ b/.github/workflows/assign_issue.yml @@ -1,10 +1,10 @@ name: Take Issue -on: - issue_comment: - types: - - created - - edited +# on: +# issue_comment: +# types: +# - created +# - edited permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml index fce770d101..356e66e6f7 100644 --- a/.github/workflows/bandit.yml +++ b/.github/workflows/bandit.yml @@ -1,9 +1,9 @@ name: python -m bandit --recursive --configfile bandit.yml . -on: - pull_request: - paths-ignore: - - 'thirdparty' - - '**.md' +# on: +# pull_request: +# paths-ignore: +# - 'thirdparty' +# - '**.md' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions jobs: bandit: diff --git a/.github/workflows/causal_lm_cpp.yml b/.github/workflows/causal_lm_cpp.yml index c75ac3214c..9d1b5564b5 100644 --- a/.github/workflows/causal_lm_cpp.yml +++ b/.github/workflows/causal_lm_cpp.yml @@ -1,12 +1,12 @@ name: causal_lm_cpp -on: - workflow_dispatch: - pull_request: - merge_group: - push: - branches: - - master - - 'releases/**' +# on: +# workflow_dispatch: +# pull_request: +# merge_group: +# push: +# branches: +# - master +# - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions diff --git a/.github/workflows/job_vlm_sample_llava.yml b/.github/workflows/job_vlm_sample_llava.yml index 7394464026..5d1ad4f59b 100644 --- a/.github/workflows/job_vlm_sample_llava.yml +++ b/.github/workflows/job_vlm_sample_llava.yml @@ -1,14 +1,14 @@ name: visual_language_chat sample - LLaVA -on: - workflow_call: - inputs: - model_id: - required: true - type: string - model_dir: - required: true - type: string +# on: +# workflow_call: +# inputs: +# model_id: +# required: true +# type: string +# model_dir: +# required: true +# type: string env: l_u22_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/l_openvino_toolkit_ubuntu22_2025.0.0.dev20241105_x86_64.tgz diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 102dee120b..1e17202ec0 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -1,6 +1,6 @@ name: "Pull Request Labeler" -on: -- pull_request_target +# on: +# - pull_request_target permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions diff --git a/.github/workflows/lcm_dreamshaper_cpp.yml b/.github/workflows/lcm_dreamshaper_cpp.yml index 6bd25cbdfe..f6c903187a 100644 --- a/.github/workflows/lcm_dreamshaper_cpp.yml +++ b/.github/workflows/lcm_dreamshaper_cpp.yml @@ -1,13 +1,13 @@ name: lcm_dreamshaper -on: - workflow_dispatch: - pull_request: - merge_group: - push: - branches: - - master - - 'releases/**' +# on: +# workflow_dispatch: +# pull_request: +# merge_group: +# push: +# branches: +# - master +# - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions @@ -46,7 +46,7 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' - + - name: Build app run: | source ${{ env.OV_INSTALL_DIR }}/setupvars.sh @@ -106,10 +106,10 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' - + - name: Create virtual environment run: python -m venv openvino_lcm_cpp - + - name: Build app run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" diff --git a/.github/workflows/llm_bench-python.yml b/.github/workflows/llm_bench-python.yml index 0ac47d1aa0..02e836349f 100644 --- a/.github/workflows/llm_bench-python.yml +++ b/.github/workflows/llm_bench-python.yml @@ -3,17 +3,17 @@ name: llm_bench Python Test -on: - push: - branches: [ "master" ] - paths: - - tools/llm_bench/** - - tools/who_what_benchmark/** - pull_request: - paths: - - tools/llm_bench/** - - tools/who_what_benchmark/** - - .github/workflows/llm_bench-python.yml +# on: +# push: +# branches: [ "master" ] +# paths: +# - tools/llm_bench/** +# - tools/who_what_benchmark/** +# pull_request: +# paths: +# - tools/llm_bench/** +# - tools/who_what_benchmark/** +# - .github/workflows/llm_bench-python.yml permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 935d6556b3..09e14f2b20 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -1,12 +1,12 @@ name: macOS (12, Python 3.9) -on: - workflow_dispatch: - pull_request: - merge_group: - push: - branches: - - master - - 'releases/**' +# on: +# workflow_dispatch: +# pull_request: +# merge_group: +# push: +# branches: +# - master +# - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions diff --git a/.github/workflows/stable_diffusion_1_5_cpp.yml b/.github/workflows/stable_diffusion_1_5_cpp.yml index f36ac43839..0c4212bfa6 100644 --- a/.github/workflows/stable_diffusion_1_5_cpp.yml +++ b/.github/workflows/stable_diffusion_1_5_cpp.yml @@ -1,13 +1,13 @@ name: stable_diffusion_1_5_cpp on: - workflow_dispatch: - pull_request: - merge_group: - push: - branches: - - master - - 'releases/**' + # workflow_dispatch: + # pull_request: + # merge_group: + # push: + # branches: + # - master + # - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 1e4164aa0b..f48e80413e 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -1,12 +1,12 @@ name: Windows (VS 2019, Python 3.11) -on: - workflow_dispatch: - pull_request: - merge_group: - push: - branches: - - master - - 'releases/**' +# on: +# workflow_dispatch: +# pull_request: +# merge_group: +# push: +# branches: +# - master +# - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions From 8632882eaa4d305b7fc05938c72f5e04e507cde8 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 26 Nov 2024 15:55:20 +0100 Subject: [PATCH 08/47] Minify linux wf --- .github/workflows/linux.yml | 458 ++++++++++++++++++------------------ 1 file changed, 229 insertions(+), 229 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 3c3e0347e7..4b3914cdb5 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -182,183 +182,183 @@ jobs: path: ${{ env.BUILD_DIR }}/openvino_package.tar.gz if-no-files-found: 'error' - genai_python_lib: - name: OpenVINO genai extension (cmake + wheel) - needs: [ openvino_download, openvino_build ] - if: | - always() && - (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') - timeout-minutes: 120 - defaults: - run: - shell: bash - runs-on: ubuntu-20.04-16-cores - env: - CMAKE_GENERATOR: Unix Makefiles - CMAKE_BUILD_PARALLEL_LEVEL: null - OV_INSTALL_DIR: ${{ github.workspace }}/ov - CCACHE_DIR: ${{ github.workspace }}/ccache - CCACHE_MAXSIZE: 500Mi - CMAKE_CXX_COMPILER_LAUNCHER: ccache - CMAKE_C_COMPILER_LAUNCHER: ccache - - steps: - - name: Clone openvino.genai - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - - - name: Download OpenVINO package - uses: actions/download-artifact@v4 - with: - name: openvino_package - path: ${{ env.OV_INSTALL_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${OV_INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${OV_INSTALL_DIR} --strip-components=1 - popd - - - name: Set apt - run: | - echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null - echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null - echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null - echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null - - - name: Install build dependencies - run: | - sudo ${OV_INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh - sudo apt-get install ccache - - - name: Setup ccache - uses: actions/cache@v4 - with: - # Should save cache only if run in the master branch of the base repo - # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push - save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} - path: ${{ env.CCACHE_DIR }} - key: ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release - - - name: Build genai - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/ - cmake --build ./build/ --config Release -j - - - name: Test bindings - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels --upgrade-strategy eager - python -m pytest -v ./tests/python_tests/test_chat_generate_api.py::test_set_chat_template - env: - PYTHONPATH: "./build/:$PYTHONPATH" - - - name: Test bindings (wheel) - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - python -m pip install . --verbose --find-links ${OV_INSTALL_DIR}/wheels - python -m pytest -v ./tests/python_tests --ignore ./tests/python_tests/test_whisper_generate_api.py --ignore ./tests/python_tests/test_vlm_api.py -k "not test_set_chat_template" - - - run: > - source ${OV_INSTALL_DIR}/setupvars.sh - && python -m pytest -v ./tests/python_tests/test_vlm_api.py - - genai_python_lib_whisper: - name: OpenVINO genai extension whisper tests (cmake + wheel) - needs: [ openvino_download, openvino_build ] - if: | - always() && - (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') - timeout-minutes: 90 - defaults: - run: - shell: bash - runs-on: ubuntu-20.04-16-cores - env: - CMAKE_GENERATOR: Unix Makefiles - CMAKE_BUILD_PARALLEL_LEVEL: null - OV_INSTALL_DIR: ${{ github.workspace }}/ov - CCACHE_DIR: ${{ github.workspace }}/ccache - CCACHE_MAXSIZE: 500Mi - CMAKE_CXX_COMPILER_LAUNCHER: ccache - CMAKE_C_COMPILER_LAUNCHER: ccache - - steps: - - name: Clone openvino.genai - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - - - name: Download OpenVINO package - uses: actions/download-artifact@v4 - with: - name: openvino_package - path: ${{ env.OV_INSTALL_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${OV_INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${OV_INSTALL_DIR} --strip-components=1 - popd - - - name: Set apt - run: | - echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null - echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null - echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null - echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null - - - name: Install build dependencies - run: | - sudo ${OV_INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh - sudo apt-get install ccache - - - name: Setup ccache - uses: actions/cache@v4 - with: - # Should save cache only if run in the master branch of the base repo - # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push - save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} - path: ${{ env.CCACHE_DIR }} - key: ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release - - - name: Build genai - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/ - cmake --build ./build/ --config Release --target py_openvino_genai -j - - - name: Test bindings - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels --upgrade-strategy eager - python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k test_smoke - env: - PYTHONPATH: "./build/:$PYTHONPATH" - - - name: Test bindings (wheel) - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - python -m pip install . --verbose --find-links ${OV_INSTALL_DIR}/wheels - python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k "not test_smoke" + # genai_python_lib: + # name: OpenVINO genai extension (cmake + wheel) + # needs: [ openvino_download, openvino_build ] + # if: | + # always() && + # (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') + # timeout-minutes: 120 + # defaults: + # run: + # shell: bash + # runs-on: ubuntu-20.04-16-cores + # env: + # CMAKE_GENERATOR: Unix Makefiles + # CMAKE_BUILD_PARALLEL_LEVEL: null + # OV_INSTALL_DIR: ${{ github.workspace }}/ov + # CCACHE_DIR: ${{ github.workspace }}/ccache + # CCACHE_MAXSIZE: 500Mi + # CMAKE_CXX_COMPILER_LAUNCHER: ccache + # CMAKE_C_COMPILER_LAUNCHER: ccache + + # steps: + # - name: Clone openvino.genai + # uses: actions/checkout@v4 + # with: + # submodules: recursive + + # - name: Setup Python ${{ env.PYTHON_VERSION }} + # uses: actions/setup-python@v5 + # with: + # python-version: ${{ env.PYTHON_VERSION }} + # cache: 'pip' + + # - name: Download OpenVINO package + # uses: actions/download-artifact@v4 + # with: + # name: openvino_package + # path: ${{ env.OV_INSTALL_DIR }} + + # - name: Extract OpenVINO packages + # run: | + # pushd ${OV_INSTALL_DIR} + # tar -xzf openvino_package.tar.gz -C ${OV_INSTALL_DIR} --strip-components=1 + # popd + + # - name: Set apt + # run: | + # echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null + # echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null + # echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null + # echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null + + # - name: Install build dependencies + # run: | + # sudo ${OV_INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh + # sudo apt-get install ccache + + # - name: Setup ccache + # uses: actions/cache@v4 + # with: + # # Should save cache only if run in the master branch of the base repo + # # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push + # save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} + # path: ${{ env.CCACHE_DIR }} + # key: ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release-${{ github.sha }} + # restore-keys: | + # ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release + + # - name: Build genai + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/ + # cmake --build ./build/ --config Release -j + + # - name: Test bindings + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels --upgrade-strategy eager + # python -m pytest -v ./tests/python_tests/test_chat_generate_api.py::test_set_chat_template + # env: + # PYTHONPATH: "./build/:$PYTHONPATH" + + # - name: Test bindings (wheel) + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # python -m pip install . --verbose --find-links ${OV_INSTALL_DIR}/wheels + # python -m pytest -v ./tests/python_tests --ignore ./tests/python_tests/test_whisper_generate_api.py --ignore ./tests/python_tests/test_vlm_api.py -k "not test_set_chat_template" + + # - run: > + # source ${OV_INSTALL_DIR}/setupvars.sh + # && python -m pytest -v ./tests/python_tests/test_vlm_api.py + + # genai_python_lib_whisper: + # name: OpenVINO genai extension whisper tests (cmake + wheel) + # needs: [ openvino_download, openvino_build ] + # if: | + # always() && + # (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') + # timeout-minutes: 90 + # defaults: + # run: + # shell: bash + # runs-on: ubuntu-20.04-16-cores + # env: + # CMAKE_GENERATOR: Unix Makefiles + # CMAKE_BUILD_PARALLEL_LEVEL: null + # OV_INSTALL_DIR: ${{ github.workspace }}/ov + # CCACHE_DIR: ${{ github.workspace }}/ccache + # CCACHE_MAXSIZE: 500Mi + # CMAKE_CXX_COMPILER_LAUNCHER: ccache + # CMAKE_C_COMPILER_LAUNCHER: ccache + + # steps: + # - name: Clone openvino.genai + # uses: actions/checkout@v4 + # with: + # submodules: recursive + + # - name: Setup Python ${{ env.PYTHON_VERSION }} + # uses: actions/setup-python@v5 + # with: + # python-version: ${{ env.PYTHON_VERSION }} + # cache: 'pip' + + # - name: Download OpenVINO package + # uses: actions/download-artifact@v4 + # with: + # name: openvino_package + # path: ${{ env.OV_INSTALL_DIR }} + + # - name: Extract OpenVINO packages + # run: | + # pushd ${OV_INSTALL_DIR} + # tar -xzf openvino_package.tar.gz -C ${OV_INSTALL_DIR} --strip-components=1 + # popd + + # - name: Set apt + # run: | + # echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null + # echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null + # echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null + # echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null + + # - name: Install build dependencies + # run: | + # sudo ${OV_INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh + # sudo apt-get install ccache + + # - name: Setup ccache + # uses: actions/cache@v4 + # with: + # # Should save cache only if run in the master branch of the base repo + # # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push + # save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} + # path: ${{ env.CCACHE_DIR }} + # key: ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release-${{ github.sha }} + # restore-keys: | + # ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release + + # - name: Build genai + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/ + # cmake --build ./build/ --config Release --target py_openvino_genai -j + + # - name: Test bindings + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels --upgrade-strategy eager + # python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k test_smoke + # env: + # PYTHONPATH: "./build/:$PYTHONPATH" + + # - name: Test bindings (wheel) + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # python -m pip install . --verbose --find-links ${OV_INSTALL_DIR}/wheels + # python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k "not test_smoke" genai_package: name: OpenVINO genai extension (install to OpenVINO package) @@ -388,11 +388,11 @@ jobs: with: submodules: recursive - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' + # - name: Setup Python ${{ env.PYTHON_VERSION }} + # uses: actions/setup-python@v5 + # with: + # python-version: ${{ env.PYTHON_VERSION }} + # cache: 'pip' - name: Download OpenVINO package uses: actions/download-artifact@v4 @@ -433,53 +433,53 @@ jobs: cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ./ -B ./build/ cmake --build ./build/ --config ${{ matrix.build-type }} --target package -j - - name: Build and Install dependencies - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --find-links ${OV_INSTALL_DIR}/wheels - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels - optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 - optimum-cli export openvino --trust-remote-code --model openai/whisper-tiny whisper-tiny - - - name: Install samples - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - cmake --install ./build/ --config ${{ matrix.build-type }} --prefix ${OV_INSTALL_DIR} - - - name: Build samples (Release) - if: ${{ 'Release' == matrix.build-type }} # build_samples enforces Release build - run: | - ${OV_INSTALL_DIR}/samples/cpp/build_samples.sh -i ${{ github.workspace }}/s\ pace - - - name: Build samples (Debug) - if: ${{ 'Release' != matrix.build-type }} - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ${OV_INSTALL_DIR}/samples/cpp/ -B ./samples\ build/ && cmake --build ./samples\ build/ --config ${{ matrix.build-type }} -j - cmake --install ./samples\ build/ --config ${{ matrix.build-type }} --component samples_bin --prefix s\ pace - - - name: Test C++ samples (greedy_causal_lm) - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - timeout 25s ${{ github.workspace }}/s\ pace/samples_bin/greedy_causal_lm ./TinyLlama-1.1B-Chat-v1.0/ "" - - - name: Test C++ samples (whisper_speech_recognition) - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - wget https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/librispeech_s5/how_are_you_doing_today.wav - timeout 25s ${{ github.workspace }}/s\ pace/samples_bin/whisper_speech_recognition ./whisper-tiny/ how_are_you_doing_today.wav - - - name: Test python samples (multinomial_causal_lm) - if: ${{ 'Release' == matrix.build-type }} # Python bindings can be built in Release only - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - timeout 25s ${OV_INSTALL_DIR}/samples/python/multinomial_causal_lm/multinomial_causal_lm.py ./TinyLlama-1.1B-Chat-v1.0/ 0 - - - name: Test python samples (whisper_speech_recognition) - if: ${{ 'Release' == matrix.build-type }} # Python bindings can be built in Release only - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - timeout 25s ${OV_INSTALL_DIR}/samples/python/whisper_speech_recognition/whisper_speech_recognition.py ./whisper-tiny/ how_are_you_doing_today.wav + # - name: Build and Install dependencies + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --find-links ${OV_INSTALL_DIR}/wheels + # python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels + # optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 + # optimum-cli export openvino --trust-remote-code --model openai/whisper-tiny whisper-tiny + + # - name: Install samples + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # cmake --install ./build/ --config ${{ matrix.build-type }} --prefix ${OV_INSTALL_DIR} + + # - name: Build samples (Release) + # if: ${{ 'Release' == matrix.build-type }} # build_samples enforces Release build + # run: | + # ${OV_INSTALL_DIR}/samples/cpp/build_samples.sh -i ${{ github.workspace }}/s\ pace + + # - name: Build samples (Debug) + # if: ${{ 'Release' != matrix.build-type }} + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ${OV_INSTALL_DIR}/samples/cpp/ -B ./samples\ build/ && cmake --build ./samples\ build/ --config ${{ matrix.build-type }} -j + # cmake --install ./samples\ build/ --config ${{ matrix.build-type }} --component samples_bin --prefix s\ pace + + # - name: Test C++ samples (greedy_causal_lm) + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # timeout 25s ${{ github.workspace }}/s\ pace/samples_bin/greedy_causal_lm ./TinyLlama-1.1B-Chat-v1.0/ "" + + # - name: Test C++ samples (whisper_speech_recognition) + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # wget https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/librispeech_s5/how_are_you_doing_today.wav + # timeout 25s ${{ github.workspace }}/s\ pace/samples_bin/whisper_speech_recognition ./whisper-tiny/ how_are_you_doing_today.wav + + # - name: Test python samples (multinomial_causal_lm) + # if: ${{ 'Release' == matrix.build-type }} # Python bindings can be built in Release only + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # timeout 25s ${OV_INSTALL_DIR}/samples/python/multinomial_causal_lm/multinomial_causal_lm.py ./TinyLlama-1.1B-Chat-v1.0/ 0 + + # - name: Test python samples (whisper_speech_recognition) + # if: ${{ 'Release' == matrix.build-type }} # Python bindings can be built in Release only + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # timeout 25s ${OV_INSTALL_DIR}/samples/python/whisper_speech_recognition/whisper_speech_recognition.py ./whisper-tiny/ how_are_you_doing_today.wav Overall_Status: name: ci/gha_overall_status_linux From 9ceb3b598a1061c6cb17351d101b9baed3f77bef Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 26 Nov 2024 16:02:26 +0100 Subject: [PATCH 09/47] Fix wf file --- .github/workflows/stable_diffusion_1_5_cpp.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/stable_diffusion_1_5_cpp.yml b/.github/workflows/stable_diffusion_1_5_cpp.yml index 0c4212bfa6..f9357902dc 100644 --- a/.github/workflows/stable_diffusion_1_5_cpp.yml +++ b/.github/workflows/stable_diffusion_1_5_cpp.yml @@ -1,13 +1,13 @@ name: stable_diffusion_1_5_cpp -on: - # workflow_dispatch: - # pull_request: - # merge_group: - # push: - # branches: - # - master - # - 'releases/**' +# on: +# workflow_dispatch: +# pull_request: +# merge_group: +# push: +# branches: +# - master +# - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions From 599713b25d0b4afad4cf0313ccd0f8d9e711849b Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 26 Nov 2024 16:05:36 +0100 Subject: [PATCH 10/47] Fix wf file --- .github/workflows/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 4b3914cdb5..d68b07ea34 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -483,7 +483,7 @@ jobs: Overall_Status: name: ci/gha_overall_status_linux - needs: [openvino_download, openvino_build, genai_python_lib, genai_package, genai_python_lib_whisper] + needs: [openvino_download, openvino_build, genai_package] if: ${{ always() }} runs-on: ubuntu-latest steps: From 222940d59df23fd3235512c9eaa8dd3a121891a9 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 26 Nov 2024 16:25:28 +0100 Subject: [PATCH 11/47] Set cpack generator as NPM --- .github/workflows/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index d68b07ea34..3a720c8f41 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -133,7 +133,7 @@ jobs: -DENABLE_STRICT_DEPENDENCIES=OFF \ -DENABLE_SYSTEM_OPENCL=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DCPACK_GENERATOR=TGZ \ + -DCPACK_GENERATOR=NPM \ -DENABLE_JS=OFF \ -DENABLE_SAMPLES=ON \ -DENABLE_OV_ONNX_FRONTEND=OFF \ From 901dd90e4e530e5293d0353da73e959fc32ec3d9 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 26 Nov 2024 17:31:50 +0100 Subject: [PATCH 12/47] Set ENABLE_JS on --- .github/workflows/linux.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 3a720c8f41..41e98a77ce 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -133,8 +133,8 @@ jobs: -DENABLE_STRICT_DEPENDENCIES=OFF \ -DENABLE_SYSTEM_OPENCL=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DCPACK_GENERATOR=NPM \ - -DENABLE_JS=OFF \ + -DCPACK_GENERATOR=TGZ \ + -DENABLE_JS=ON \ -DENABLE_SAMPLES=ON \ -DENABLE_OV_ONNX_FRONTEND=OFF \ -DENABLE_OV_PADDLE_FRONTEND=OFF \ @@ -430,7 +430,7 @@ jobs: - name: Build genai run: | source ${OV_INSTALL_DIR}/setupvars.sh - cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ./ -B ./build/ + cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -DENABLE_JS=ON -S ./ -B ./build/ cmake --build ./build/ --config ${{ matrix.build-type }} --target package -j # - name: Build and Install dependencies From 09356efaf528ac608d8acfdf43f2bfd79432dade Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 26 Nov 2024 17:44:59 +0100 Subject: [PATCH 13/47] Add genai_node_addon component if ENABLE_JS is ON --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 35ca895abc..5ffe7fbdf1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,6 +102,9 @@ set(CPACK_COMPONENTS_ALL core_genai core_genai_dev cpp_samples_genai licensing_g if(ENABLE_PYTHON) list(APPEND CPACK_COMPONENTS_ALL pygenai_${Python3_VERSION_MAJOR}_${Python3_VERSION_MINOR}) endif() +if(ENABLE_JS) + list(APPEND CPACK_COMPONENTS_ALL genai_node_addon) +endif() if(WIN32 AND NOT DEFINED CPACK_GENERATOR) set(CPACK_GENERATOR "ZIP") endif() From 96d53b2aed1bb978f2ebbf82b63a4ab75c9b244f Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 27 Nov 2024 14:04:09 +0100 Subject: [PATCH 14/47] Add wf to check genai js bindings compilation --- .github/workflows/linux.yml | 388 ++---------------------------------- 1 file changed, 17 insertions(+), 371 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 41e98a77ce..00cbd187aa 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -51,8 +51,8 @@ jobs: path: openvino_package.tar.gz if-no-files-found: 'error' - openvino_build: - name: Build OpenVINO package + genai_nodejs_bindings: + name: Produce genai nodejs binaries archive needs: [openvino_download] if: needs.openvino_download.outputs.status != 'success' timeout-minutes: 150 @@ -88,30 +88,11 @@ jobs: submodules: 'true' ref: ${{ env.OV_BRANCH}} - # - # Dependencies - # - - name: Install build dependencies run: | sudo -E ${OPENVINO_REPO}/install_build_dependencies.sh sudo apt-get install ccache - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - - - name: Install python dependencies - run: | - # For Python API: build and wheel packaging - python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt - - # - # Build - # - - name: Setup ccache uses: actions/cache@v4 with: @@ -119,371 +100,36 @@ jobs: # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} path: ${{ env.CCACHE_DIR }} - key: ${{ runner.os }}-${{ runner.arch }}-ccache-ov-${{ github.sha }} + key: ${{ runner.os }}-${{ runner.arch }}-ccache-ov-and-genai-${{ matrix.build-type }}-${{ github.sha }} restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-ccache-ov + ${{ runner.os }}-${{ runner.arch }}-ccache-ov-and-genai-${{ matrix.build-type }} - - name: CMake configure - OpenVINO + - name: Build openvino + genai run: | - cmake \ - -G "${{ env.CMAKE_GENERATOR }}" \ - -DENABLE_CPPLINT=OFF \ - -DENABLE_NCC_STYLE=OFF \ - -DENABLE_TESTS=OFF \ - -DENABLE_STRICT_DEPENDENCIES=OFF \ - -DENABLE_SYSTEM_OPENCL=ON \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DCPACK_GENERATOR=TGZ \ - -DENABLE_JS=ON \ - -DENABLE_SAMPLES=ON \ - -DENABLE_OV_ONNX_FRONTEND=OFF \ - -DENABLE_OV_PADDLE_FRONTEND=OFF \ - -DENABLE_OV_PYTORCH_FRONTEND=ON \ - -DENABLE_OV_TF_FRONTEND=ON \ - -DENABLE_OV_TF_LITE_FRONTEND=OFF \ - -DENABLE_INTEL_GPU=OFF \ - -DENABLE_INTEL_NPU=ON \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DENABLE_PYTHON=ON \ - -DENABLE_WHEEL=ON \ - -S ${OPENVINO_REPO} \ - -B ${BUILD_DIR} - - - name: Clean ccache stats - run: ccache --zero-stats --show-config - - - name: Cmake build - OpenVINO - run: cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} - - - name: Show ccache stats - run: ccache --show-stats - - - name: Cmake install - OpenVINO - run: | - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/openvino_package -P ${BUILD_DIR}/cmake_install.cmake - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/openvino_package -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake - - - name: Pack Artifacts - run: | - pushd ${INSTALL_DIR} - tar -czvf ${BUILD_DIR}/openvino_package.tar.gz * - popd + source ${OV_INSTALL_DIR}/setupvars.sh + cmake -DOPENVINO_EXTRA_MODULES=./ -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ + -DCPACK_GENERATOR=NPM -UTBB* -DENABLE_SYSTEM_TBB=OFF \ + -DENABLE_WHEEL=OFF \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/openvino_package \ + -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ + -S ${OPENVINO_REPO} -B ./build + cmake --build ./build/ --target package -j # # Upload build artifacts and logs # - - name: Upload openvino package + - name: Upload genai nodejs bindings archive if: ${{ always() }} uses: actions/upload-artifact@v4 with: - name: openvino_package - path: ${{ env.BUILD_DIR }}/openvino_package.tar.gz + name: genai_nodejs_bindings + path: ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz if-no-files-found: 'error' - # genai_python_lib: - # name: OpenVINO genai extension (cmake + wheel) - # needs: [ openvino_download, openvino_build ] - # if: | - # always() && - # (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') - # timeout-minutes: 120 - # defaults: - # run: - # shell: bash - # runs-on: ubuntu-20.04-16-cores - # env: - # CMAKE_GENERATOR: Unix Makefiles - # CMAKE_BUILD_PARALLEL_LEVEL: null - # OV_INSTALL_DIR: ${{ github.workspace }}/ov - # CCACHE_DIR: ${{ github.workspace }}/ccache - # CCACHE_MAXSIZE: 500Mi - # CMAKE_CXX_COMPILER_LAUNCHER: ccache - # CMAKE_C_COMPILER_LAUNCHER: ccache - - # steps: - # - name: Clone openvino.genai - # uses: actions/checkout@v4 - # with: - # submodules: recursive - - # - name: Setup Python ${{ env.PYTHON_VERSION }} - # uses: actions/setup-python@v5 - # with: - # python-version: ${{ env.PYTHON_VERSION }} - # cache: 'pip' - - # - name: Download OpenVINO package - # uses: actions/download-artifact@v4 - # with: - # name: openvino_package - # path: ${{ env.OV_INSTALL_DIR }} - - # - name: Extract OpenVINO packages - # run: | - # pushd ${OV_INSTALL_DIR} - # tar -xzf openvino_package.tar.gz -C ${OV_INSTALL_DIR} --strip-components=1 - # popd - - # - name: Set apt - # run: | - # echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null - # echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null - # echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null - # echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null - - # - name: Install build dependencies - # run: | - # sudo ${OV_INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh - # sudo apt-get install ccache - - # - name: Setup ccache - # uses: actions/cache@v4 - # with: - # # Should save cache only if run in the master branch of the base repo - # # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push - # save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} - # path: ${{ env.CCACHE_DIR }} - # key: ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release-${{ github.sha }} - # restore-keys: | - # ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release - - # - name: Build genai - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/ - # cmake --build ./build/ --config Release -j - - # - name: Test bindings - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels --upgrade-strategy eager - # python -m pytest -v ./tests/python_tests/test_chat_generate_api.py::test_set_chat_template - # env: - # PYTHONPATH: "./build/:$PYTHONPATH" - - # - name: Test bindings (wheel) - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # python -m pip install . --verbose --find-links ${OV_INSTALL_DIR}/wheels - # python -m pytest -v ./tests/python_tests --ignore ./tests/python_tests/test_whisper_generate_api.py --ignore ./tests/python_tests/test_vlm_api.py -k "not test_set_chat_template" - - # - run: > - # source ${OV_INSTALL_DIR}/setupvars.sh - # && python -m pytest -v ./tests/python_tests/test_vlm_api.py - - # genai_python_lib_whisper: - # name: OpenVINO genai extension whisper tests (cmake + wheel) - # needs: [ openvino_download, openvino_build ] - # if: | - # always() && - # (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') - # timeout-minutes: 90 - # defaults: - # run: - # shell: bash - # runs-on: ubuntu-20.04-16-cores - # env: - # CMAKE_GENERATOR: Unix Makefiles - # CMAKE_BUILD_PARALLEL_LEVEL: null - # OV_INSTALL_DIR: ${{ github.workspace }}/ov - # CCACHE_DIR: ${{ github.workspace }}/ccache - # CCACHE_MAXSIZE: 500Mi - # CMAKE_CXX_COMPILER_LAUNCHER: ccache - # CMAKE_C_COMPILER_LAUNCHER: ccache - - # steps: - # - name: Clone openvino.genai - # uses: actions/checkout@v4 - # with: - # submodules: recursive - - # - name: Setup Python ${{ env.PYTHON_VERSION }} - # uses: actions/setup-python@v5 - # with: - # python-version: ${{ env.PYTHON_VERSION }} - # cache: 'pip' - - # - name: Download OpenVINO package - # uses: actions/download-artifact@v4 - # with: - # name: openvino_package - # path: ${{ env.OV_INSTALL_DIR }} - - # - name: Extract OpenVINO packages - # run: | - # pushd ${OV_INSTALL_DIR} - # tar -xzf openvino_package.tar.gz -C ${OV_INSTALL_DIR} --strip-components=1 - # popd - - # - name: Set apt - # run: | - # echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null - # echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null - # echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null - # echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null - - # - name: Install build dependencies - # run: | - # sudo ${OV_INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh - # sudo apt-get install ccache - - # - name: Setup ccache - # uses: actions/cache@v4 - # with: - # # Should save cache only if run in the master branch of the base repo - # # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push - # save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} - # path: ${{ env.CCACHE_DIR }} - # key: ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release-${{ github.sha }} - # restore-keys: | - # ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release - - # - name: Build genai - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/ - # cmake --build ./build/ --config Release --target py_openvino_genai -j - - # - name: Test bindings - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels --upgrade-strategy eager - # python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k test_smoke - # env: - # PYTHONPATH: "./build/:$PYTHONPATH" - - # - name: Test bindings (wheel) - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # python -m pip install . --verbose --find-links ${OV_INSTALL_DIR}/wheels - # python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k "not test_smoke" - - genai_package: - name: OpenVINO genai extension (install to OpenVINO package) - strategy: - matrix: - build-type: [Release, Debug] - needs: [ openvino_download, openvino_build ] - if: | - always() && - (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') - timeout-minutes: 60 - defaults: - run: - shell: bash - runs-on: ubuntu-20.04 - env: - CMAKE_BUILD_PARALLEL_LEVEL: null - OV_INSTALL_DIR: ${{ github.workspace }}/ov - CCACHE_DIR: ${{ github.workspace }}/ccache - CCACHE_MAXSIZE: 500Mi - CMAKE_CXX_COMPILER_LAUNCHER: ccache - CMAKE_C_COMPILER_LAUNCHER: ccache - - steps: - - name: Clone openvino.genai - uses: actions/checkout@v4 - with: - submodules: recursive - - # - name: Setup Python ${{ env.PYTHON_VERSION }} - # uses: actions/setup-python@v5 - # with: - # python-version: ${{ env.PYTHON_VERSION }} - # cache: 'pip' - - - name: Download OpenVINO package - uses: actions/download-artifact@v4 - with: - name: openvino_package - path: ${{ env.OV_INSTALL_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${OV_INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${OV_INSTALL_DIR} --strip-components=1 - popd - - - name: Set apt - run: | - echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null - echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null - echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null - echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null - - - name: Install build dependencies - run: | - sudo ${OV_INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh - sudo apt-get install ccache - - - name: Setup ccache - uses: actions/cache@v4 - with: - save-always: true - path: ${{ env.CCACHE_DIR }} - key: ${{ runner.os }}-${{ runner.arch }}-ccache-genai-${{ matrix.build-type }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-ccache-genai-${{ matrix.build-type }} - - - name: Build genai - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -DENABLE_JS=ON -S ./ -B ./build/ - cmake --build ./build/ --config ${{ matrix.build-type }} --target package -j - - # - name: Build and Install dependencies - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --find-links ${OV_INSTALL_DIR}/wheels - # python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels - # optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 - # optimum-cli export openvino --trust-remote-code --model openai/whisper-tiny whisper-tiny - - # - name: Install samples - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # cmake --install ./build/ --config ${{ matrix.build-type }} --prefix ${OV_INSTALL_DIR} - - # - name: Build samples (Release) - # if: ${{ 'Release' == matrix.build-type }} # build_samples enforces Release build - # run: | - # ${OV_INSTALL_DIR}/samples/cpp/build_samples.sh -i ${{ github.workspace }}/s\ pace - - # - name: Build samples (Debug) - # if: ${{ 'Release' != matrix.build-type }} - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ${OV_INSTALL_DIR}/samples/cpp/ -B ./samples\ build/ && cmake --build ./samples\ build/ --config ${{ matrix.build-type }} -j - # cmake --install ./samples\ build/ --config ${{ matrix.build-type }} --component samples_bin --prefix s\ pace - - # - name: Test C++ samples (greedy_causal_lm) - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # timeout 25s ${{ github.workspace }}/s\ pace/samples_bin/greedy_causal_lm ./TinyLlama-1.1B-Chat-v1.0/ "" - - # - name: Test C++ samples (whisper_speech_recognition) - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # wget https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/librispeech_s5/how_are_you_doing_today.wav - # timeout 25s ${{ github.workspace }}/s\ pace/samples_bin/whisper_speech_recognition ./whisper-tiny/ how_are_you_doing_today.wav - - # - name: Test python samples (multinomial_causal_lm) - # if: ${{ 'Release' == matrix.build-type }} # Python bindings can be built in Release only - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # timeout 25s ${OV_INSTALL_DIR}/samples/python/multinomial_causal_lm/multinomial_causal_lm.py ./TinyLlama-1.1B-Chat-v1.0/ 0 - - # - name: Test python samples (whisper_speech_recognition) - # if: ${{ 'Release' == matrix.build-type }} # Python bindings can be built in Release only - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # timeout 25s ${OV_INSTALL_DIR}/samples/python/whisper_speech_recognition/whisper_speech_recognition.py ./whisper-tiny/ how_are_you_doing_today.wav - Overall_Status: name: ci/gha_overall_status_linux - needs: [openvino_download, openvino_build, genai_package] + needs: [openvino_download, genai_nodejs_bindings] if: ${{ always() }} runs-on: ubuntu-latest steps: From 785bf8d90973acaba18a94614c323dcef605ecf2 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 27 Nov 2024 14:09:45 +0100 Subject: [PATCH 15/47] Remove setupvars.sh run --- .github/workflows/linux.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 00cbd187aa..4fc5caf3cf 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -106,7 +106,6 @@ jobs: - name: Build openvino + genai run: | - source ${OV_INSTALL_DIR}/setupvars.sh cmake -DOPENVINO_EXTRA_MODULES=./ -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ -DCPACK_GENERATOR=NPM -UTBB* -DENABLE_SYSTEM_TBB=OFF \ -DENABLE_WHEEL=OFF \ From e8c53f286c8a083a08c806bb4cbca61c49ada1b9 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 27 Nov 2024 15:37:00 +0100 Subject: [PATCH 16/47] Fix ov + genai workflow --- .github/workflows/linux.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 4fc5caf3cf..b334defe9b 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -66,10 +66,9 @@ jobs: CMAKE_GENERATOR: 'Ninja Multi-Config' CMAKE_CXX_COMPILER_LAUNCHER: ccache CMAKE_C_COMPILER_LAUNCHER: ccache - OPENVINO_REPO: ${{ github.workspace }}/openvino - INSTALL_DIR: ${{ github.workspace }}/openvino/install - BUILD_DIR: ${{ github.workspace }}/openvino/build - CCACHE_DIR: ${{ github.workspace }}/ccache + OPENVINO_REPO: ${{ github.workspace }}/../openvino + BUILD_DIR: ${{ github.workspace }}/../build + CCACHE_DIR: ${{ github.workspace }}/../ccache CCACHE_MAXSIZE: 2000Mi steps: @@ -106,13 +105,12 @@ jobs: - name: Build openvino + genai run: | - cmake -DOPENVINO_EXTRA_MODULES=./ -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ + cd .. && cmake -DOPENVINO_EXTRA_MODULES=openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ -DCPACK_GENERATOR=NPM -UTBB* -DENABLE_SYSTEM_TBB=OFF \ -DENABLE_WHEEL=OFF \ - -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/openvino_package \ -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ - -S ${OPENVINO_REPO} -B ./build - cmake --build ./build/ --target package -j + -S openvino -B ./build + cmake --build ./build --target package -j # # Upload build artifacts and logs @@ -123,7 +121,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: genai_nodejs_bindings - path: ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz + path: ../build/genai_nodejs_bindings.tar.gz if-no-files-found: 'error' Overall_Status: From 82ae75f4f68850676ae9e5dd1d9db773a26f7cc0 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 27 Nov 2024 15:47:22 +0100 Subject: [PATCH 17/47] Fix ov + genai workflow --- .github/workflows/linux.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index b334defe9b..ec6dab48da 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -66,9 +66,10 @@ jobs: CMAKE_GENERATOR: 'Ninja Multi-Config' CMAKE_CXX_COMPILER_LAUNCHER: ccache CMAKE_C_COMPILER_LAUNCHER: ccache - OPENVINO_REPO: ${{ github.workspace }}/../openvino - BUILD_DIR: ${{ github.workspace }}/../build - CCACHE_DIR: ${{ github.workspace }}/../ccache + OPENVINO_REPO: ${{ github.workspace }}/openvino + GENAI_REPO: ${{ github.workspace }}/openvino.genai + BUILD_DIR: ${{ github.workspace }}/build + CCACHE_DIR: ${{ github.workspace }}/ccache CCACHE_MAXSIZE: 2000Mi steps: @@ -92,6 +93,12 @@ jobs: sudo -E ${OPENVINO_REPO}/install_build_dependencies.sh sudo apt-get install ccache + - name: Clone GenAI + uses: actions/checkout@v4 + with: + path: ${{ env.GENAI_REPO }} + submodules: recursive + - name: Setup ccache uses: actions/cache@v4 with: @@ -105,11 +112,11 @@ jobs: - name: Build openvino + genai run: | - cd .. && cmake -DOPENVINO_EXTRA_MODULES=openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ + cmake -DOPENVINO_EXTRA_MODULES=./openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ -DCPACK_GENERATOR=NPM -UTBB* -DENABLE_SYSTEM_TBB=OFF \ -DENABLE_WHEEL=OFF \ -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ - -S openvino -B ./build + -S ./openvino -B ./build cmake --build ./build --target package -j # @@ -121,7 +128,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: genai_nodejs_bindings - path: ../build/genai_nodejs_bindings.tar.gz + path: ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz if-no-files-found: 'error' Overall_Status: From 963fc1d3446fe0ccd206cbe7474e615c83fe7024 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 27 Nov 2024 17:43:01 +0100 Subject: [PATCH 18/47] Fix path to genai folder --- .github/workflows/linux.yml | 44 +++++++++---------------------------- 1 file changed, 10 insertions(+), 34 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index ec6dab48da..9ccd38ed9f 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -17,44 +17,13 @@ concurrency: env: PYTHON_VERSION: '3.9' - OV_BRANCH: 0080d90974ca84f9a6d359da3388a2a18a93b753 + OV_BRANCH: 1c9b23ccca1cca72e3f11f8cdad2e88fc7f7e633 OV_TARBALL: '' jobs: - openvino_download: - name: Download OpenVINO package - outputs: - status: ${{ steps.openvino_download.outcome }} - timeout-minutes: 10 - defaults: - run: - shell: bash - runs-on: ubuntu-20.04 - - steps: - - name: Download OpenVINO build - id: openvino_download - run: | - wget ${{ env.OV_TARBALL}} --progress=bar:force:noscroll -O openvino_package.tar.gz - tar -tvf openvino_package.tar.gz - continue-on-error: true - - # - # Upload to artifacts - # - - - name: Upload openvino package - if: steps.openvino_download.outcome == 'success' - uses: actions/upload-artifact@v4 - with: - name: openvino_package - path: openvino_package.tar.gz - if-no-files-found: 'error' genai_nodejs_bindings: name: Produce genai nodejs binaries archive - needs: [openvino_download] - if: needs.openvino_download.outputs.status != 'success' timeout-minutes: 150 defaults: run: @@ -99,6 +68,12 @@ jobs: path: ${{ env.GENAI_REPO }} submodules: recursive + - name: Check GenAI repo contents + run: | + ls -la ${{ env.GENAI_REPO }}/src/js + cd ${{ env.GENAI_REPO }} + git log + - name: Setup ccache uses: actions/cache@v4 with: @@ -112,8 +87,9 @@ jobs: - name: Build openvino + genai run: | - cmake -DOPENVINO_EXTRA_MODULES=./openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ + cmake -DOPENVINO_EXTRA_MODULES=../openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ -DCPACK_GENERATOR=NPM -UTBB* -DENABLE_SYSTEM_TBB=OFF \ + -DENABLE_PYTHON=OFF \ -DENABLE_WHEEL=OFF \ -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ -S ./openvino -B ./build @@ -133,7 +109,7 @@ jobs: Overall_Status: name: ci/gha_overall_status_linux - needs: [openvino_download, genai_nodejs_bindings] + needs: [genai_nodejs_bindings] if: ${{ always() }} runs-on: ubuntu-latest steps: From 1d952b6d7d8d8c34a831e786627c1d84db752476 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Thu, 28 Nov 2024 10:29:32 +0100 Subject: [PATCH 19/47] Change OV branch hash --- .github/workflows/linux.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 9ccd38ed9f..3d7b1fd751 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -17,7 +17,7 @@ concurrency: env: PYTHON_VERSION: '3.9' - OV_BRANCH: 1c9b23ccca1cca72e3f11f8cdad2e88fc7f7e633 + OV_BRANCH: 0080d90974ca84f9a6d359da3388a2a18a93b753 OV_TARBALL: '' jobs: @@ -68,12 +68,6 @@ jobs: path: ${{ env.GENAI_REPO }} submodules: recursive - - name: Check GenAI repo contents - run: | - ls -la ${{ env.GENAI_REPO }}/src/js - cd ${{ env.GENAI_REPO }} - git log - - name: Setup ccache uses: actions/cache@v4 with: From e909f03ecfa371a10e74b07e3b81246f583a5029 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 11 Dec 2024 14:48:20 +0100 Subject: [PATCH 20/47] Align openvino_tokenizers version with master --- thirdparty/openvino_tokenizers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thirdparty/openvino_tokenizers b/thirdparty/openvino_tokenizers index aeb1bc2cb1..904046825b 160000 --- a/thirdparty/openvino_tokenizers +++ b/thirdparty/openvino_tokenizers @@ -1 +1 @@ -Subproject commit aeb1bc2cb1646650d6ff0fa258ca775bd20796d6 +Subproject commit 904046825b6378bae74f16f302b40599aa88d5b3 From c5f5f85d7035c409dfe2478a522f7eb38996c33e Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 11 Dec 2024 14:49:06 +0100 Subject: [PATCH 21/47] Move rpath setting to libopenvino_genai part --- src/cpp/CMakeLists.txt | 13 +++++++++++++ src/js/CMakeLists.txt | 13 ------------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt index 308ac119f4..ef719ff1be 100644 --- a/src/cpp/CMakeLists.txt +++ b/src/cpp/CMakeLists.txt @@ -138,6 +138,19 @@ if(CPACK_GENERATOR STREQUAL "NPM") set(LIBRARY_DESTINATION .) set(ARCHIVE_DESTINATION .) set(RUNTIME_DESTINATION .) + + # setting RPATH / LC_RPATH depending on platform + if(LINUX) + # to find libopenvino_genai.so in the same folder + set(rpaths "$ORIGIN") + elseif(APPLE) + # to find libopenvino_genai.dylib in the same folder + set(rpaths "@loader_path") + endif() + + if(rpaths) + set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "${rpaths}") + endif() else() set(LIBRARY_DESTINATION runtime/lib/${ARCH_DIR}) set(ARCHIVE_DESTINATION runtime/lib/${ARCH_DIR}) diff --git a/src/js/CMakeLists.txt b/src/js/CMakeLists.txt index d7ea811618..275803a804 100644 --- a/src/js/CMakeLists.txt +++ b/src/js/CMakeLists.txt @@ -73,19 +73,6 @@ set_target_properties(${PROJECT_NAME} PROPERTIES SUFFIX ".node" ) -# setting RPATH / LC_RPATH depending on platform -if(LINUX) - # to find libopenvino_genai.so in the same folder - set(rpaths "$ORIGIN") -elseif(APPLE) - # to find libopenvino_genai.dylib in the same folder - set(rpaths "@loader_path") -endif() - -if(rpaths) - set_target_properties(${PROJECT_NAME} PROPERTIES INSTALL_RPATH "${rpaths}") -endif() - install(TARGETS ${PROJECT_NAME} LIBRARY DESTINATION . COMPONENT ${PROJECT_NAME} RUNTIME DESTINATION . COMPONENT ${PROJECT_NAME} From 6d7cbcd8722c3ce1df3b3f3c0bc84bd84ebb429e Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 11 Dec 2024 15:22:00 +0100 Subject: [PATCH 22/47] Add ENABLE_JS option to features.cmake --- .github/workflows/linux.yml | 2 +- cmake/features.cmake | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 3d7b1fd751..7e87a0e992 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -82,7 +82,7 @@ jobs: - name: Build openvino + genai run: | cmake -DOPENVINO_EXTRA_MODULES=../openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ - -DCPACK_GENERATOR=NPM -UTBB* -DENABLE_SYSTEM_TBB=OFF \ + -DCPACK_GENERATOR=NPM -DENABLE_JS=ON -UTBB* -DENABLE_SYSTEM_TBB=OFF \ -DENABLE_PYTHON=OFF \ -DENABLE_WHEEL=OFF \ -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ diff --git a/cmake/features.cmake b/cmake/features.cmake index 0434b21ee9..999ee1d81e 100644 --- a/cmake/features.cmake +++ b/cmake/features.cmake @@ -3,3 +3,4 @@ # option(ENABLE_PYTHON "Enable Python API build" ON) +option(ENABLE_JS "Enable JS API build" OFF) From d99956977f206660b9bf9f6ee9632f663a5a6b8a Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 11 Dec 2024 15:29:30 +0100 Subject: [PATCH 23/47] Rollback worflow files --- .github/workflows/assign_issue.yml | 10 +- .github/workflows/bandit.yml | 10 +- .github/workflows/causal_lm_cpp.yml | 117 ++--- .github/workflows/job_vlm_sample_llava.yml | 20 +- .github/workflows/labeler.yml | 4 +- .github/workflows/lcm_dreamshaper_cpp.yml | 86 ++-- .github/workflows/linux.yml | 426 +++++++++++++++++- .github/workflows/llm_bench-python.yml | 51 +-- .github/workflows/mac.yml | 24 +- .../workflows/stable_diffusion_1_5_cpp.yml | 60 +-- .github/workflows/windows.yml | 26 +- 11 files changed, 629 insertions(+), 205 deletions(-) diff --git a/.github/workflows/assign_issue.yml b/.github/workflows/assign_issue.yml index 68bfb6cf2d..4a4579e2c7 100644 --- a/.github/workflows/assign_issue.yml +++ b/.github/workflows/assign_issue.yml @@ -1,10 +1,10 @@ name: Take Issue -# on: -# issue_comment: -# types: -# - created -# - edited +on: + issue_comment: + types: + - created + - edited permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml index 356e66e6f7..fce770d101 100644 --- a/.github/workflows/bandit.yml +++ b/.github/workflows/bandit.yml @@ -1,9 +1,9 @@ name: python -m bandit --recursive --configfile bandit.yml . -# on: -# pull_request: -# paths-ignore: -# - 'thirdparty' -# - '**.md' +on: + pull_request: + paths-ignore: + - 'thirdparty' + - '**.md' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions jobs: bandit: diff --git a/.github/workflows/causal_lm_cpp.yml b/.github/workflows/causal_lm_cpp.yml index 9d1b5564b5..504e303fb5 100644 --- a/.github/workflows/causal_lm_cpp.yml +++ b/.github/workflows/causal_lm_cpp.yml @@ -1,12 +1,12 @@ name: causal_lm_cpp -# on: -# workflow_dispatch: -# pull_request: -# merge_group: -# push: -# branches: -# - master -# - 'releases/**' +on: + workflow_dispatch: + pull_request: + merge_group: + push: + branches: + - master + - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions @@ -16,10 +16,10 @@ concurrency: cancel-in-progress: true env: - l_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/l_openvino_toolkit_ubuntu20_2025.0.0.dev20241105_x86_64.tgz - l_u22_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/l_openvino_toolkit_ubuntu22_2025.0.0.dev20241105_x86_64.tgz - m_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/m_openvino_toolkit_macos_12_6_2025.0.0.dev20241105_x86_64.tgz - w_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/w_openvino_toolkit_windows_2025.0.0.dev20241105_x86_64.zip + l_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17539-6abe2e39391/l_openvino_toolkit_ubuntu20_2025.0.0.dev20241205_x86_64.tgz + l_u22_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17539-6abe2e39391/l_openvino_toolkit_ubuntu22_2025.0.0.dev20241205_x86_64.tgz + m_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17539-6abe2e39391/m_openvino_toolkit_macos_12_6_2025.0.0.dev20241205_x86_64.tgz + w_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17539-6abe2e39391/w_openvino_toolkit_windows_2025.0.0.dev20241205_x86_64.zip jobs: cpp-multinomial-greedy_causal_lm-ubuntu: runs-on: ubuntu-20.04-8-cores @@ -46,8 +46,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model openlm-research/open_llama_3b_v2 open_llama_3b_v2 optimum-cli export openvino -m TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T wget https://huggingface.co/smangrul/tinyllama_lora_sql/resolve/main/adapter_model.safetensors?download=true -O adapter_model.safetensors @@ -105,8 +105,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 - name: Compare env: @@ -118,7 +118,7 @@ jobs: import transformers with open('pred.txt', 'r') as file: predictions = file.read() - tokenizer = transformers.LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') + tokenizer = transformers.AutoTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') tokenized = tokenizer('Why is the Sun yellow?', return_tensors='pt') for beam in transformers.LlamaForCausalLM.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0').generate(**tokenized, num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9, do_sample=False): ref = ': ' + tokenizer.decode(beam[tokenized['input_ids'].numel():], skip_special_tokens=True) @@ -134,7 +134,7 @@ jobs: import transformers with open('pred.txt', 'r') as file: predictions = file.read() - tokenizer = transformers.LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') + tokenizer = transformers.AutoTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') tokenized = tokenizer('69', return_tensors='pt') for beam in transformers.LlamaForCausalLM.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0').generate(**tokenized, num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9, do_sample=False): ref = ': ' + tokenizer.decode(beam[tokenized['input_ids'].numel():], skip_special_tokens=True) @@ -150,7 +150,7 @@ jobs: import transformers with open('pred.txt', 'r') as file: predictions = file.read() - tokenizer = transformers.LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') + tokenizer = transformers.AutoTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') tokenized = tokenizer('Hi', return_tensors='pt') for beam in transformers.LlamaForCausalLM.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0').generate(**tokenized, num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9, do_sample=False): ref = ': ' + tokenizer.decode(beam[tokenized['input_ids'].numel():], skip_special_tokens=True) @@ -166,7 +166,7 @@ jobs: import transformers with open('pred.txt', 'r') as file: predictions = file.read() - tokenizer = transformers.LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') + tokenizer = transformers.AutoTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') tokenized = tokenizer('return 0', return_tensors='pt') for beam in transformers.LlamaForCausalLM.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0').generate(**tokenized, num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9, do_sample=False): ref = ': ' + tokenizer.decode(beam[tokenized['input_ids'].numel():], skip_special_tokens=True) @@ -182,7 +182,7 @@ jobs: import transformers with open('pred.txt', 'r', errors='ignore') as file: predictions = file.read() - tokenizer = transformers.LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') + tokenizer = transformers.AutoTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') tokenized = tokenizer('你好！ 你好嗎？', return_tensors='pt') for beam in transformers.LlamaForCausalLM.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0').generate(**tokenized, num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9, do_sample=False): ref = ': ' + tokenizer.decode(beam[tokenized['input_ids'].numel():], skip_special_tokens=True) @@ -198,7 +198,7 @@ jobs: import transformers with open('pred.txt', 'r', errors='ignore') as file: predictions = file.read() - tokenizer = transformers.LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') + tokenizer = transformers.AutoTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') prompts = [ 'Alan Turing was a', 'return 0', @@ -241,8 +241,8 @@ jobs: - name: Download and convert model run: | call .\ov\setupvars.bat - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 optimum-cli export openvino -m TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T curl -o adapter_model.safetensors -s -L https://huggingface.co/smangrul/tinyllama_lora_sql/resolve/main/adapter_model.safetensors?download=true @@ -299,8 +299,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model Qwen/Qwen-7B-Chat Qwen-7B-Chat - run: > . ./ov/setupvars.sh @@ -333,8 +333,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model Qwen/Qwen1.5-7B-Chat Qwen1.5-7B-Chat - run: > . ./ov/setupvars.sh @@ -368,8 +368,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model microsoft/phi-2 phi-2 - run: > . ./ov/setupvars.sh @@ -403,8 +403,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model argilla/notus-7b-v1 notus-7b-v1 - run: > . ./ov/setupvars.sh @@ -438,8 +438,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model databricks/dolly-v2-3b dolly-v2-3b optimum-cli export openvino --trust-remote-code --weight-format fp16 --model databricks/dolly-v2-7b dolly-v2-7b - name: run and compare @@ -488,8 +488,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 optimum-cli export openvino --trust-remote-code --weight-format fp16 --model Qwen/Qwen-7B-Chat Qwen-7B-Chat --task text-generation-with-past - name: run and compare @@ -560,8 +560,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model microsoft/phi-1_5 phi-1_5 - name: Run Generation run: | @@ -591,7 +591,7 @@ jobs: PYTHONPATH: "./build" cpp-greedy_causal_lm-redpajama-3b-chat: - runs-on: ubuntu-20.04-4-cores + runs-on: ubuntu-20.04-8-cores defaults: run: shell: bash @@ -615,8 +615,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model ikala/redpajama-3b-chat redpajama-3b-chat - name: Run Generation run: | @@ -670,8 +670,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 - name: Compare env: @@ -681,9 +681,9 @@ jobs: printf 'What is 2 + 2?\nWhat is the previous answer?\nAdd 1 to it.\nSubtract 5 from it.\nWhy is the sun yellow?\nWhat was my first question?\n' > ./input.txt timeout 30s ./build/samples/cpp/chat_sample/chat_sample ./TinyLlama-1.1B-Chat-v1.0/ < input.txt > ./pred.txt python -c " - from transformers import LlamaTokenizer, AutoModelForCausalLM + from transformers import AutoTokenizer, AutoModelForCausalLM model_id = 'TinyLlama/TinyLlama-1.1B-Chat-v1.0' - tokenizer = LlamaTokenizer.from_pretrained(model_id) + tokenizer = AutoTokenizer.from_pretrained(model_id) model = AutoModelForCausalLM.from_pretrained(model_id) prompts = ['What is 2 + 2?', 'What is the previous answer?', 'Add 1 to it.', 'Subtract 5 from it.', 'Why is the sun yellow?', 'What was my first question?'] def gen_prompt(prompt): @@ -727,7 +727,7 @@ jobs: ov_link: ${{ env.l_u22_ov_link }} - uses: ./.github/actions/build_app with: - build_target: 'visual_language_chat py_openvino_genai' + build_target: 'visual_language_chat benchmark_vlm py_openvino_genai' - uses: ./.github/actions/install_python_deps - name: Download and convert tiny-random-minicpmv-2_6 model and an image run: | @@ -754,6 +754,12 @@ jobs: && ./build/samples/cpp/visual_language_chat/visual_language_chat ./tiny-random-minicpmv-2_6/ ./images/ <<< $'Describe the images?' | tee cpp.txt timeout-minutes: 2 + - name: Run benchmark_vlm C++ sample - tiny-random-minicpmv-2_6 + run: > + set -o pipefail + && source ./ov/setupvars.sh + && ./build/samples/cpp/visual_language_chat/benchmark_vlm -m ./tiny-random-minicpmv-2_6/ -i ./images/cat.png -n 3 + timeout-minutes: 2 - name: Run visual_language_chat Python sample - tiny-random-minicpmv-2_6 run: > set -o pipefail @@ -762,6 +768,13 @@ jobs: <<< $'Describe the images?' | tee py.txt env: PYTHONPATH: "./build/" + - name: Run benchmark_vlm Python sample - tiny-random-minicpmv-2_6 + run: > + set -o pipefail + && source ./ov/setupvars.sh + && ./samples/python/visual_language_chat/benchmark_vlm.py -m ./tiny-random-minicpmv-2_6/ -i ./images/cat.png -n 3 + env: + PYTHONPATH: "./build/" - name: Encode cpp.txt with Python encoding instead of terminal one shell: python run: | @@ -863,8 +876,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 - name: Run gtests run: | @@ -909,8 +922,8 @@ jobs: - name: Download and convert and model run: | call .\ov\setupvars.bat - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 - name: Run gtests run: | @@ -954,8 +967,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 - name: Run gtests run: | diff --git a/.github/workflows/job_vlm_sample_llava.yml b/.github/workflows/job_vlm_sample_llava.yml index 5d1ad4f59b..166284bd4b 100644 --- a/.github/workflows/job_vlm_sample_llava.yml +++ b/.github/workflows/job_vlm_sample_llava.yml @@ -1,17 +1,17 @@ name: visual_language_chat sample - LLaVA -# on: -# workflow_call: -# inputs: -# model_id: -# required: true -# type: string -# model_dir: -# required: true -# type: string +on: + workflow_call: + inputs: + model_id: + required: true + type: string + model_dir: + required: true + type: string env: - l_u22_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/l_openvino_toolkit_ubuntu22_2025.0.0.dev20241105_x86_64.tgz + l_u22_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17539-6abe2e39391/l_openvino_toolkit_ubuntu22_2025.0.0.dev20241205_x86_64.tgz jobs: visual_language_chat_sample-ubuntu-llava: diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 1e17202ec0..102dee120b 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -1,6 +1,6 @@ name: "Pull Request Labeler" -# on: -# - pull_request_target +on: +- pull_request_target permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions diff --git a/.github/workflows/lcm_dreamshaper_cpp.yml b/.github/workflows/lcm_dreamshaper_cpp.yml index f6c903187a..b3a36761e1 100644 --- a/.github/workflows/lcm_dreamshaper_cpp.yml +++ b/.github/workflows/lcm_dreamshaper_cpp.yml @@ -1,13 +1,13 @@ name: lcm_dreamshaper -# on: -# workflow_dispatch: -# pull_request: -# merge_group: -# push: -# branches: -# - master -# - 'releases/**' +on: + workflow_dispatch: + pull_request: + merge_group: + push: + branches: + - master + - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions @@ -18,8 +18,8 @@ concurrency: env: PYTHON_VERSION: '3.9' - LINUX_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/l_openvino_toolkit_ubuntu22_2025.0.0.dev20241105_x86_64.tgz - WINDOWS_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/w_openvino_toolkit_windows_2025.0.0.dev20241105_x86_64.zip + LINUX_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17539-6abe2e39391/l_openvino_toolkit_ubuntu22_2025.0.0.dev20241205_x86_64.tgz + WINDOWS_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17539-6abe2e39391/w_openvino_toolkit_windows_2025.0.0.dev20241205_x86_64.zip OV_INSTALL_DIR: ${{ github.workspace }}/ov jobs: @@ -46,12 +46,12 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' - - - name: Build app + + - name: Build apps run: | source ${{ env.OV_INSTALL_DIR }}/setupvars.sh cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ${{ env.build_dir }} - cmake --build ${{ env.build_dir }} --config Release --target stable_diffusion heterogeneous_stable_diffusion lora_stable_diffusion py_openvino_genai --parallel + cmake --build ${{ env.build_dir }} --config Release --target text2image image2image inpainting heterogeneous_stable_diffusion lora_text2image py_openvino_genai --parallel - name: Create virtual environment run: python3 -m venv openvino_lcm_cpp @@ -59,27 +59,39 @@ jobs: - name: Install python dependencies run: | source openvino_lcm_cpp/bin/activate - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly python -m pip install -r ./samples/requirements.txt - name: Download and convert models and tokenizer run: | source openvino_lcm_cpp/bin/activate optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --task stable-diffusion --weight-format fp16 models/lcm_dreamshaper_v7/FP16 + wget -O ./image.png https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png + wget -O ./mask_image.png https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png - - name: Run app + - name: Run heterogeneous_stable_diffusion run: | source ${{ env.OV_INSTALL_DIR }}/setupvars.sh - ${{ env.build_dir }}/samples/cpp/text2image/heterogeneous_stable_diffusion ./models/lcm_dreamshaper_v7/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" + ${{ env.build_dir }}/samples/cpp/image_generation/heterogeneous_stable_diffusion ./models/lcm_dreamshaper_v7/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" - - name: Run Python app + - name: Run heterogeneous_stable_diffusion.py run: | source openvino_lcm_cpp/bin/activate source ./ov/setupvars.sh - python ./samples/python/text2image/heterogeneous_stable_diffusion.py ./models/lcm_dreamshaper_v7/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" + python ./samples/python/image_generation/heterogeneous_stable_diffusion.py ./models/lcm_dreamshaper_v7/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" env: PYTHONPATH: ${{ env.build_dir }} + - name: Run image2image + run: | + source ./ov/setupvars.sh + ${{ env.build_dir }}/samples/cpp/image_generation/image2image ./models/lcm_dreamshaper_v7/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" ./image.png + + - name: Run inpainting + run: | + source ./ov/setupvars.sh + ${{ env.build_dir }}/samples/cpp/image_generation/inpainting ./models/lcm_dreamshaper_v7/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" ./image.png ./mask_image.png + lcm_dreamshaper_v7_cpp-windows: runs-on: windows-2019 defaults: @@ -106,38 +118,58 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' - + - name: Create virtual environment run: python -m venv openvino_lcm_cpp - - - name: Build app + + - name: Build apps run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ${{ env.build_dir }} - cmake --build ${{ env.build_dir }} --config Release --target stable_diffusion heterogeneous_stable_diffusion lora_stable_diffusion py_openvino_genai --parallel + cmake --build ${{ env.build_dir }} --config Release --target text2image image2image inpainting heterogeneous_stable_diffusion lora_text2image py_openvino_genai --parallel - name: Install python dependencies run: | . "./openvino_lcm_cpp/Scripts/Activate.ps1" - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly python -m pip install -r ./samples/requirements.txt - name: Download and convert models and tokenizer run: | . "./openvino_lcm_cpp/Scripts/Activate.ps1" optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --task stable-diffusion --weight-format fp16 models/lcm_dreamshaper_v7/FP16 + Invoke-WebRequest -Uri 'https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png' -OutFile 'image.png' + Invoke-WebRequest -Uri 'https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png' -OutFile 'mask_image.png' - - name: Run app + - name: Run heterogeneous_stable_diffusion run: > . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" - & "${{ env.build_dir }}/samples/cpp/text2image/Release/heterogeneous_stable_diffusion.exe ./models/lcm_dreamshaper_v7/FP16 'cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting'" + & "${{ env.build_dir }}/samples/cpp/image_generation/Release/heterogeneous_stable_diffusion.exe ./models/lcm_dreamshaper_v7/FP16 'cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting'" + + - name: Run heterogeneous_stable_diffusion.py + run: | + . "./openvino_lcm_cpp/Scripts/Activate.ps1" + . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" + $env:Path += "${{ env.build_dir }}\openvino_genai" + python .\samples\python\image_generation\heterogeneous_stable_diffusion.py .\models\lcm_dreamshaper_v7\FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" + env: + PYTHONPATH: ${{ env.build_dir }} + + - name: Run image2image.py + run: | + . "./openvino_lcm_cpp/Scripts/Activate.ps1" + . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" + $env:Path += "${{ env.build_dir }}\openvino_genai" + python .\samples\python\image_generation\image2image.py .\models\lcm_dreamshaper_v7\FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" .\image.png + env: + PYTHONPATH: ${{ env.build_dir }} - - name: Run Python app + - name: Run inpainting.py run: | . "./openvino_lcm_cpp/Scripts/Activate.ps1" . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" $env:Path += "${{ env.build_dir }}\openvino_genai" - python .\samples\python\text2image\heterogeneous_stable_diffusion.py .\models\lcm_dreamshaper_v7\FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" + python .\samples\python\image_generation\inpainting.py .\models\lcm_dreamshaper_v7\FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" .\image.png .\mask_image.png env: PYTHONPATH: ${{ env.build_dir }} diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 7e87a0e992..2916c340bf 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -21,9 +21,40 @@ env: OV_TARBALL: '' jobs: + openvino_download: + name: Download OpenVINO package + outputs: + status: ${{ steps.openvino_download.outcome }} + timeout-minutes: 10 + defaults: + run: + shell: bash + runs-on: ubuntu-20.04 + + steps: + - name: Download OpenVINO build + id: openvino_download + run: | + wget ${{ env.OV_TARBALL}} --progress=bar:force:noscroll -O openvino_package.tar.gz + tar -tvf openvino_package.tar.gz + continue-on-error: true - genai_nodejs_bindings: - name: Produce genai nodejs binaries archive + # + # Upload to artifacts + # + + - name: Upload openvino package + if: steps.openvino_download.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: openvino_package + path: openvino_package.tar.gz + if-no-files-found: 'error' + + openvino_build: + name: Build OpenVINO package + needs: [openvino_download] + if: needs.openvino_download.outputs.status != 'success' timeout-minutes: 150 defaults: run: @@ -36,8 +67,8 @@ jobs: CMAKE_CXX_COMPILER_LAUNCHER: ccache CMAKE_C_COMPILER_LAUNCHER: ccache OPENVINO_REPO: ${{ github.workspace }}/openvino - GENAI_REPO: ${{ github.workspace }}/openvino.genai - BUILD_DIR: ${{ github.workspace }}/build + INSTALL_DIR: ${{ github.workspace }}/openvino/install + BUILD_DIR: ${{ github.workspace }}/openvino/build CCACHE_DIR: ${{ github.workspace }}/ccache CCACHE_MAXSIZE: 2000Mi @@ -57,16 +88,29 @@ jobs: submodules: 'true' ref: ${{ env.OV_BRANCH}} + # + # Dependencies + # + - name: Install build dependencies run: | sudo -E ${OPENVINO_REPO}/install_build_dependencies.sh sudo apt-get install ccache - - name: Clone GenAI - uses: actions/checkout@v4 + - name: Setup Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 with: - path: ${{ env.GENAI_REPO }} - submodules: recursive + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + + - name: Install python dependencies + run: | + # For Python API: build and wheel packaging + python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt + + # + # Build + # - name: Setup ccache uses: actions/cache@v4 @@ -75,35 +119,373 @@ jobs: # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} path: ${{ env.CCACHE_DIR }} - key: ${{ runner.os }}-${{ runner.arch }}-ccache-ov-and-genai-${{ matrix.build-type }}-${{ github.sha }} + key: ${{ runner.os }}-${{ runner.arch }}-ccache-ov-${{ github.sha }} restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-ccache-ov-and-genai-${{ matrix.build-type }} + ${{ runner.os }}-${{ runner.arch }}-ccache-ov - - name: Build openvino + genai + - name: CMake configure - OpenVINO run: | - cmake -DOPENVINO_EXTRA_MODULES=../openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ - -DCPACK_GENERATOR=NPM -DENABLE_JS=ON -UTBB* -DENABLE_SYSTEM_TBB=OFF \ - -DENABLE_PYTHON=OFF \ - -DENABLE_WHEEL=OFF \ - -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ - -S ./openvino -B ./build - cmake --build ./build --target package -j + cmake \ + -G "${{ env.CMAKE_GENERATOR }}" \ + -DENABLE_CPPLINT=OFF \ + -DENABLE_NCC_STYLE=OFF \ + -DENABLE_TESTS=OFF \ + -DENABLE_STRICT_DEPENDENCIES=OFF \ + -DENABLE_SYSTEM_OPENCL=ON \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCPACK_GENERATOR=TGZ \ + -DENABLE_JS=OFF \ + -DENABLE_SAMPLES=ON \ + -DENABLE_OV_ONNX_FRONTEND=OFF \ + -DENABLE_OV_PADDLE_FRONTEND=OFF \ + -DENABLE_OV_PYTORCH_FRONTEND=ON \ + -DENABLE_OV_TF_FRONTEND=ON \ + -DENABLE_OV_TF_LITE_FRONTEND=OFF \ + -DENABLE_INTEL_GPU=OFF \ + -DENABLE_INTEL_NPU=ON \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DENABLE_PYTHON=ON \ + -DENABLE_WHEEL=ON \ + -S ${OPENVINO_REPO} \ + -B ${BUILD_DIR} + + - name: Clean ccache stats + run: ccache --zero-stats --show-config + + - name: Cmake build - OpenVINO + run: cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + + - name: Show ccache stats + run: ccache --show-stats + + - name: Cmake install - OpenVINO + run: | + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/openvino_package -P ${BUILD_DIR}/cmake_install.cmake + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/openvino_package -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake + + - name: Pack Artifacts + run: | + pushd ${INSTALL_DIR} + tar -czvf ${BUILD_DIR}/openvino_package.tar.gz * + popd # # Upload build artifacts and logs # - - name: Upload genai nodejs bindings archive + - name: Upload openvino package if: ${{ always() }} uses: actions/upload-artifact@v4 with: - name: genai_nodejs_bindings - path: ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz + name: openvino_package + path: ${{ env.BUILD_DIR }}/openvino_package.tar.gz if-no-files-found: 'error' + genai_python_lib: + name: OpenVINO genai extension (cmake + wheel) + needs: [ openvino_download, openvino_build ] + if: | + always() && + (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') + timeout-minutes: 120 + defaults: + run: + shell: bash + runs-on: ubuntu-20.04-16-cores + env: + CMAKE_GENERATOR: Unix Makefiles + CMAKE_BUILD_PARALLEL_LEVEL: null + OV_INSTALL_DIR: ${{ github.workspace }}/ov + CCACHE_DIR: ${{ github.workspace }}/ccache + CCACHE_MAXSIZE: 500Mi + CMAKE_CXX_COMPILER_LAUNCHER: ccache + CMAKE_C_COMPILER_LAUNCHER: ccache + + steps: + - name: Clone openvino.genai + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + + - name: Download OpenVINO package + uses: actions/download-artifact@v4 + with: + name: openvino_package + path: ${{ env.OV_INSTALL_DIR }} + + - name: Extract OpenVINO packages + run: | + pushd ${OV_INSTALL_DIR} + tar -xzf openvino_package.tar.gz -C ${OV_INSTALL_DIR} --strip-components=1 + popd + + - name: Set apt + run: | + echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null + echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null + echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null + echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null + + - name: Install build dependencies + run: | + sudo ${OV_INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh + sudo apt-get install ccache + + - name: Setup ccache + uses: actions/cache@v4 + with: + # Should save cache only if run in the master branch of the base repo + # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push + save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} + path: ${{ env.CCACHE_DIR }} + key: ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release + + - name: Build genai + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/ + cmake --build ./build/ --config Release -j + + - name: Test bindings + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels + python -m pytest -v ./tests/python_tests/test_chat_generate_api.py::test_set_chat_template + env: + PYTHONPATH: "./build/:$PYTHONPATH" + + - name: Test bindings (wheel) + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + python -m pip install . --verbose --find-links ${OV_INSTALL_DIR}/wheels + python -m pip install ./tools/who_what_benchmark --find-links ${OV_INSTALL_DIR}/wheels + python -m pytest -v ./tests/python_tests --ignore ./tests/python_tests/test_whisper_generate_api.py --ignore ./tests/python_tests/test_vlm_api.py -k "not test_set_chat_template" + + - run: > + source ${OV_INSTALL_DIR}/setupvars.sh + && python -m pytest -v ./tests/python_tests/test_vlm_api.py + + genai_python_lib_whisper: + name: OpenVINO genai extension whisper tests (cmake + wheel) + needs: [ openvino_download, openvino_build ] + if: | + always() && + (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') + timeout-minutes: 90 + defaults: + run: + shell: bash + runs-on: ubuntu-20.04-16-cores + env: + CMAKE_GENERATOR: Unix Makefiles + CMAKE_BUILD_PARALLEL_LEVEL: null + OV_INSTALL_DIR: ${{ github.workspace }}/ov + CCACHE_DIR: ${{ github.workspace }}/ccache + CCACHE_MAXSIZE: 500Mi + CMAKE_CXX_COMPILER_LAUNCHER: ccache + CMAKE_C_COMPILER_LAUNCHER: ccache + + steps: + - name: Clone openvino.genai + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + + - name: Download OpenVINO package + uses: actions/download-artifact@v4 + with: + name: openvino_package + path: ${{ env.OV_INSTALL_DIR }} + + - name: Extract OpenVINO packages + run: | + pushd ${OV_INSTALL_DIR} + tar -xzf openvino_package.tar.gz -C ${OV_INSTALL_DIR} --strip-components=1 + popd + + - name: Set apt + run: | + echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null + echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null + echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null + echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null + + - name: Install build dependencies + run: | + sudo ${OV_INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh + sudo apt-get install ccache + + - name: Setup ccache + uses: actions/cache@v4 + with: + # Should save cache only if run in the master branch of the base repo + # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push + save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} + path: ${{ env.CCACHE_DIR }} + key: ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release + + - name: Build genai + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/ + cmake --build ./build/ --config Release --target py_openvino_genai -j + + - name: Test bindings + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels + python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k test_smoke + env: + PYTHONPATH: "./build/:$PYTHONPATH" + + - name: Test bindings (wheel) + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + python -m pip install . --verbose --find-links ${OV_INSTALL_DIR}/wheels + python -m pip install ./tools/who_what_benchmark --find-links ${OV_INSTALL_DIR}/wheels + python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k "not test_smoke" + + genai_package: + name: OpenVINO genai extension (install to OpenVINO package) + strategy: + matrix: + build-type: [Release, Debug] + needs: [ openvino_download, openvino_build ] + if: | + always() && + (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') + timeout-minutes: 60 + defaults: + run: + shell: bash + runs-on: ubuntu-20.04 + env: + CMAKE_BUILD_PARALLEL_LEVEL: null + OV_INSTALL_DIR: ${{ github.workspace }}/ov + CCACHE_DIR: ${{ github.workspace }}/ccache + CCACHE_MAXSIZE: 500Mi + CMAKE_CXX_COMPILER_LAUNCHER: ccache + CMAKE_C_COMPILER_LAUNCHER: ccache + + steps: + - name: Clone openvino.genai + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + + - name: Download OpenVINO package + uses: actions/download-artifact@v4 + with: + name: openvino_package + path: ${{ env.OV_INSTALL_DIR }} + + - name: Extract OpenVINO packages + run: | + pushd ${OV_INSTALL_DIR} + tar -xzf openvino_package.tar.gz -C ${OV_INSTALL_DIR} --strip-components=1 + popd + + - name: Set apt + run: | + echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null + echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null + echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null + echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null + + - name: Install build dependencies + run: | + sudo ${OV_INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh + sudo apt-get install ccache + + - name: Setup ccache + uses: actions/cache@v4 + with: + save-always: true + path: ${{ env.CCACHE_DIR }} + key: ${{ runner.os }}-${{ runner.arch }}-ccache-genai-${{ matrix.build-type }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-ccache-genai-${{ matrix.build-type }} + + - name: Build genai + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ./ -B ./build/ + cmake --build ./build/ --config ${{ matrix.build-type }} --target package -j + + - name: Build and Install dependencies + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --find-links ${OV_INSTALL_DIR}/wheels + python -m pip install -r ./samples/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels + optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 + optimum-cli export openvino --trust-remote-code --model openai/whisper-tiny whisper-tiny + + - name: Install samples + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + cmake --install ./build/ --config ${{ matrix.build-type }} --prefix ${OV_INSTALL_DIR} + + - name: Build samples (Release) + if: ${{ 'Release' == matrix.build-type }} # build_samples enforces Release build + run: | + ${OV_INSTALL_DIR}/samples/cpp/build_samples.sh -i ${{ github.workspace }}/s\ pace + + - name: Build samples (Debug) + if: ${{ 'Release' != matrix.build-type }} + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ${OV_INSTALL_DIR}/samples/cpp/ -B ./samples\ build/ && cmake --build ./samples\ build/ --config ${{ matrix.build-type }} -j + cmake --install ./samples\ build/ --config ${{ matrix.build-type }} --component samples_bin --prefix s\ pace + + - name: Test C++ samples (greedy_causal_lm) + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + timeout 25s ${{ github.workspace }}/s\ pace/samples_bin/greedy_causal_lm ./TinyLlama-1.1B-Chat-v1.0/ "" + + - name: Test C++ samples (whisper_speech_recognition) + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + wget https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/librispeech_s5/how_are_you_doing_today.wav + timeout 25s ${{ github.workspace }}/s\ pace/samples_bin/whisper_speech_recognition ./whisper-tiny/ how_are_you_doing_today.wav + + - name: Test python samples (multinomial_causal_lm) + if: ${{ 'Release' == matrix.build-type }} # Python bindings can be built in Release only + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + timeout 25s ${OV_INSTALL_DIR}/samples/python/multinomial_causal_lm/multinomial_causal_lm.py ./TinyLlama-1.1B-Chat-v1.0/ 0 + + - name: Test python samples (whisper_speech_recognition) + if: ${{ 'Release' == matrix.build-type }} # Python bindings can be built in Release only + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + timeout 25s ${OV_INSTALL_DIR}/samples/python/whisper_speech_recognition/whisper_speech_recognition.py ./whisper-tiny/ how_are_you_doing_today.wav + Overall_Status: name: ci/gha_overall_status_linux - needs: [genai_nodejs_bindings] + needs: [openvino_download, openvino_build, genai_python_lib, genai_package, genai_python_lib_whisper] if: ${{ always() }} runs-on: ubuntu-latest steps: diff --git a/.github/workflows/llm_bench-python.yml b/.github/workflows/llm_bench-python.yml index 02e836349f..2c8f6a358a 100644 --- a/.github/workflows/llm_bench-python.yml +++ b/.github/workflows/llm_bench-python.yml @@ -3,17 +3,14 @@ name: llm_bench Python Test -# on: -# push: -# branches: [ "master" ] -# paths: -# - tools/llm_bench/** -# - tools/who_what_benchmark/** -# pull_request: -# paths: -# - tools/llm_bench/** -# - tools/who_what_benchmark/** -# - .github/workflows/llm_bench-python.yml +on: + workflow_dispatch: + pull_request: + merge_group: + push: + branches: + - master + - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions @@ -66,28 +63,28 @@ jobs: python ./tools/llm_bench/benchmark.py -m tiny-random-qwen -d cpu -n 1 -f pt env: GIT_LFS_SKIP_SMUDGE: 0 - - name: Test tiny-random-baichuan2 on Linux + - name: Test tiny-random-baichuan2 on Linux Optimum Intel run: | optimum-cli export openvino --model katuni4ka/tiny-random-baichuan2 --trust-remote-code --weight-format fp16 ./ov_models/tiny-random-baichuan2/pytorch/dldt/FP16 - python ./tools/llm_bench/benchmark.py -m ./ov_models/tiny-random-baichuan2/pytorch/dldt/FP16/ -d cpu -n 1 - - name: Test tiny-stable-diffusion on Linux + python ./tools/llm_bench/benchmark.py -m ./ov_models/tiny-random-baichuan2/pytorch/dldt/FP16/ -d cpu -n 1 --optimum + - name: Test tiny-stable-diffusion on Linux Optimum Intel run: | optimum-cli export openvino --model segmind/tiny-sd --trust-remote-code --weight-format fp16 ./ov_models/tiny-sd/pytorch/dldt/FP16/ - python ./tools/llm_bench/benchmark.py -m ./ov_models/tiny-sd/pytorch/dldt/FP16/ -pf ./tools/llm_bench/prompts/stable-diffusion.jsonl -d cpu -n 1 + python ./tools/llm_bench/benchmark.py -m ./ov_models/tiny-sd/pytorch/dldt/FP16/ -pf ./tools/llm_bench/prompts/stable-diffusion.jsonl -d cpu -n 1 --optimum - name: Test dreamlike-anime on Linux with GenAI run: | optimum-cli export openvino --model dreamlike-art/dreamlike-anime-1.0 --task stable-diffusion --weight-format fp16 ov_models/dreamlike-art-dreamlike-anime-1.0/FP16 - python ./tools/llm_bench/benchmark.py -m ./ov_models/dreamlike-art-dreamlike-anime-1.0/FP16/ -pf ./tools/llm_bench/prompts/stable-diffusion.jsonl -d cpu -n 1 --genai + python ./tools/llm_bench/benchmark.py -m ./ov_models/dreamlike-art-dreamlike-anime-1.0/FP16/ -pf ./tools/llm_bench/prompts/stable-diffusion.jsonl -d cpu -n 1 - name: Test dreamlike-anime on Linux with GenAI and LoRA run: | wget -O ./ov_models/soulcard.safetensors https://civitai.com/api/download/models/72591 - python ./tools/llm_bench/benchmark.py -m ./ov_models/dreamlike-art-dreamlike-anime-1.0/FP16/ -pf ./tools/llm_bench/prompts/stable-diffusion.jsonl -d cpu -n 1 --genai --lora ./ov_models/soulcard.safetensors --lora_alphas 0.7 + python ./tools/llm_bench/benchmark.py -m ./ov_models/dreamlike-art-dreamlike-anime-1.0/FP16/ -pf ./tools/llm_bench/prompts/stable-diffusion.jsonl -d cpu -n 1 --lora ./ov_models/soulcard.safetensors --lora_alphas 0.7 - name: Test TinyLlama-1.1B-Chat-v1.0 in Speculative Deconding mode on Linux run: | optimum-cli export openvino --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 --trust-remote-code --weight-format fp16 ov_models/TinyLlama-1.1B-Chat-v1.0/FP16 optimum-cli export openvino --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 --trust-remote-code --weight-format int8 ov_models/TinyLlama-1.1B-Chat-v1.0/INT8 - python ./tools/llm_bench/benchmark.py -m ./ov_models/TinyLlama-1.1B-Chat-v1.0/FP16/ --draft_model ./ov_models/TinyLlama-1.1B-Chat-v1.0/INT8/ -p "Why is the Sun yellow?" -d cpu --draft_device cpu -n 1 --genai --assistant_confidence_threshold 0.4 - python ./tools/llm_bench/benchmark.py -m ./ov_models/TinyLlama-1.1B-Chat-v1.0/FP16/ --draft_model ./ov_models/TinyLlama-1.1B-Chat-v1.0/INT8/ -p "Why is the Sun yellow?" -d cpu --draft_device cpu -n 1 --genai --num_assistant_tokens 5 + python ./tools/llm_bench/benchmark.py -m ./ov_models/TinyLlama-1.1B-Chat-v1.0/FP16/ --draft_model ./ov_models/TinyLlama-1.1B-Chat-v1.0/INT8/ -p "Why is the Sun yellow?" -d cpu --draft_device cpu -n 1 --assistant_confidence_threshold 0.4 + python ./tools/llm_bench/benchmark.py -m ./ov_models/TinyLlama-1.1B-Chat-v1.0/FP16/ --draft_model ./ov_models/TinyLlama-1.1B-Chat-v1.0/INT8/ -p "Why is the Sun yellow?" -d cpu --draft_device cpu -n 1 --num_assistant_tokens 5 - name: Test whisper-tiny on Linux run: | GIT_LFS_SKIP_SMUDGE=1 git clone --depth 1 --branch main --single-branch https://huggingface.co/datasets/facebook/multilingual_librispeech @@ -97,15 +94,13 @@ jobs: tar zxvf data/mls_polish/train/audio/3283_1447_000.tar.gz -C data/mls_polish/train/audio/3283_1447_000/ cd .. optimum-cli export openvino --trust-remote-code --model openai/whisper-tiny ./ov_models/whisper-tiny + python ./tools/llm_bench/benchmark.py -m ./ov_models/whisper-tiny --media multilingual_librispeech/data/mls_polish/train/audio/3283_1447_000/3283_1447_000000.flac -d cpu -n 1 --optimum python ./tools/llm_bench/benchmark.py -m ./ov_models/whisper-tiny --media multilingual_librispeech/data/mls_polish/train/audio/3283_1447_000/3283_1447_000000.flac -d cpu -n 1 - python ./tools/llm_bench/benchmark.py -m ./ov_models/whisper-tiny --media multilingual_librispeech/data/mls_polish/train/audio/3283_1447_000/3283_1447_000000.flac -d cpu -n 1 --genai - name: WWB Tests run: | - GIT_CLONE_PROTECTION_ACTIVE=false pip install -r ${{ env.WWB_PATH }}/requirements.txt pip install git+https://github.com/huggingface/optimum-intel.git - GIT_CLONE_PROTECTION_ACTIVE=false pip install ${{ env.WWB_PATH }} - python -m pip install -U --pre openvino openvino-tokenizers openvino-genai --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly --force-reinstall - python -m pytest -v tools/who_what_benchmark/tests + GIT_CLONE_PROTECTION_ACTIVE=false PIP_PRE=1 PIP_EXTRA_INDEX_URL=https://storage.openvinotoolkit.org/simple/wheels/nightly pip install ${{ env.WWB_PATH }} + python -m pytest -v ${{ env.WWB_PATH }}/tests stateful: runs-on: ubuntu-20.04 steps: @@ -122,9 +117,7 @@ jobs: grep beam_idx pytorch/dldt/FP32/openvino_model.xml - name: WWB Tests run: | - GIT_CLONE_PROTECTION_ACTIVE=false pip install -r tools/who_what_benchmark/requirements.txt - pip install git+https://github.com/huggingface/optimum-intel.git - GIT_CLONE_PROTECTION_ACTIVE=false pip install tools/who_what_benchmark/ pip install pytest - python -m pip install -U --pre openvino openvino-tokenizers openvino-genai --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly --force-reinstall - python -m pytest -v tools/who_what_benchmark/tests + pip install git+https://github.com/huggingface/optimum-intel.git + GIT_CLONE_PROTECTION_ACTIVE=false PIP_PRE=1 PIP_EXTRA_INDEX_URL=https://storage.openvinotoolkit.org/simple/wheels/nightly pip install ${{ env.WWB_PATH }} + python -m pytest -v ${{ env.WWB_PATH }}/tests diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 09e14f2b20..7a4ee31beb 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -1,12 +1,12 @@ name: macOS (12, Python 3.9) -# on: -# workflow_dispatch: -# pull_request: -# merge_group: -# push: -# branches: -# - master -# - 'releases/**' +on: + workflow_dispatch: + pull_request: + merge_group: + push: + branches: + - master + - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions @@ -224,7 +224,7 @@ jobs: - name: Test bindings run: | source ${OV_INSTALL_DIR}/setupvars.sh - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels --upgrade-strategy eager + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels python -m pytest -v ./tests/python_tests/test_chat_generate_api.py::test_set_chat_template env: PYTHONPATH: "./build/:$PYTHONPATH" @@ -234,6 +234,7 @@ jobs: source ${OV_INSTALL_DIR}/setupvars.sh python -m pip install . --verbose --find-links ${OV_INSTALL_DIR}/wheels python -c "from openvino_genai import LLMPipeline" + python -m pip install ./tools/who_what_benchmark --find-links ${OV_INSTALL_DIR}/wheels python -m pytest -v ./tests/python_tests/ --ignore ./tests/python_tests/test_whisper_generate_api.py --ignore ./tests/python_tests/test_vlm_api.py -k "not test_set_chat_template" genai_python_lib_whisper: @@ -288,7 +289,7 @@ jobs: - name: Test bindings run: | source ${OV_INSTALL_DIR}/setupvars.sh - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels --upgrade-strategy eager + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k test_smoke env: PYTHONPATH: "./build/:$PYTHONPATH" @@ -298,6 +299,7 @@ jobs: source ${OV_INSTALL_DIR}/setupvars.sh python -m pip install . --verbose --find-links ${OV_INSTALL_DIR}/wheels python -c "from openvino_genai import LLMPipeline" + python -m pip install ./tools/who_what_benchmark --find-links ${OV_INSTALL_DIR}/wheels python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k "not test_smoke" genai_package: @@ -354,7 +356,7 @@ jobs: run: | source ${OV_INSTALL_DIR}/setupvars.sh python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --find-links ${OV_INSTALL_DIR}/wheels - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels + python -m pip install -r ./samples/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 optimum-cli export openvino --trust-remote-code --model openai/whisper-tiny whisper-tiny diff --git a/.github/workflows/stable_diffusion_1_5_cpp.yml b/.github/workflows/stable_diffusion_1_5_cpp.yml index f9357902dc..b355cd4f09 100644 --- a/.github/workflows/stable_diffusion_1_5_cpp.yml +++ b/.github/workflows/stable_diffusion_1_5_cpp.yml @@ -1,13 +1,13 @@ name: stable_diffusion_1_5_cpp -# on: -# workflow_dispatch: -# pull_request: -# merge_group: -# push: -# branches: -# - master -# - 'releases/**' +on: + workflow_dispatch: + pull_request: + merge_group: + push: + branches: + - master + - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions @@ -18,8 +18,8 @@ concurrency: env: PYTHON_VERSION: '3.10' - LINUX_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/l_openvino_toolkit_ubuntu20_2025.0.0.dev20241105_x86_64.tgz - WINDOWS_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/w_openvino_toolkit_windows_2025.0.0.dev20241105_x86_64.zip + LINUX_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17539-6abe2e39391/l_openvino_toolkit_ubuntu20_2025.0.0.dev20241205_x86_64.tgz + WINDOWS_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17539-6abe2e39391/w_openvino_toolkit_windows_2025.0.0.dev20241205_x86_64.zip OV_INSTALL_DIR: ${{ github.workspace }}/ov jobs: @@ -51,7 +51,7 @@ jobs: run: | source ${{ env.OV_INSTALL_DIR }}/setupvars.sh cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ${{ env.build_dir }} - cmake --build ${{ env.build_dir }} --config Release --target stable_diffusion lora_stable_diffusion py_openvino_genai --parallel + cmake --build ${{ env.build_dir }} --config Release --target text2image image2image inpainting heterogeneous_stable_diffusion lora_text2image py_openvino_genai --parallel - name: Create virtual environment run: python3 -m venv openvino_sd_cpp @@ -59,7 +59,7 @@ jobs: - name: Install python dependencies run: | source openvino_sd_cpp/bin/activate - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly python -m pip install -r ./samples/requirements.txt - name: Download and convert models and tokenizer @@ -68,29 +68,29 @@ jobs: optimum-cli export openvino --model dreamlike-art/dreamlike-anime-1.0 --weight-format fp16 --task stable-diffusion models/dreamlike-art-dreamlike-anime-1.0/FP16 wget -O ./models/soulcard.safetensors https://civitai.com/api/download/models/72591 - - name: Run main app + - name: Run text2image app run: | source ${{ env.OV_INSTALL_DIR }}/setupvars.sh - ${{ env.build_dir }}/samples/cpp/text2image/stable_diffusion ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" + ${{ env.build_dir }}/samples/cpp/image_generation/text2image ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" - - name: Run LoRA app + - name: Run lora_text2image app run: | source ${{ env.OV_INSTALL_DIR }}/setupvars.sh - ${{ env.build_dir }}/samples/cpp/text2image/lora_stable_diffusion ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "curly-haired unicorn in the forest, anime, line" ./models/soulcard.safetensors 0.7 + ${{ env.build_dir }}/samples/cpp/image_generation/lora_text2image ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "curly-haired unicorn in the forest, anime, line" ./models/soulcard.safetensors 0.7 - - name: Run Python main app + - name: Run text2image.py app run: | source openvino_sd_cpp/bin/activate source ./ov/setupvars.sh - python ./samples/python/text2image/main.py ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" + python ./samples/python/image_generation/text2image.py ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" env: PYTHONPATH: ${{ env.build_dir }} - - name: Run Python LoRA app + - name: Run lora_text2image.py app run: | source openvino_sd_cpp/bin/activate source ./ov/setupvars.sh - python ./samples/python/text2image/lora.py ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "curly-haired unicorn in the forest, anime, line" ./models/soulcard.safetensors 0.7 + python ./samples/python/image_generation/lora_text2image.py ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "curly-haired unicorn in the forest, anime, line" ./models/soulcard.safetensors 0.7 env: PYTHONPATH: ${{ env.build_dir }} @@ -125,7 +125,7 @@ jobs: run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ${{ env.build_dir }} - cmake --build ${{ env.build_dir }} --config Release --target stable_diffusion lora_stable_diffusion py_openvino_genai --parallel + cmake --build ${{ env.build_dir }} --config Release --target text2image image2image inpainting heterogeneous_stable_diffusion lora_text2image py_openvino_genai --parallel - name: Create virtual environment run: python -m venv openvino_sd_cpp @@ -133,7 +133,7 @@ jobs: - name: Install python dependencies run: | . "./openvino_sd_cpp/Scripts/Activate.ps1" - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly python -m pip install -r ./samples/requirements.txt - name: Download and convert models and tokenizer @@ -142,35 +142,35 @@ jobs: optimum-cli export openvino --model dreamlike-art/dreamlike-anime-1.0 --task stable-diffusion --weight-format fp16 models/dreamlike-art-dreamlike-anime-1.0/FP16 Invoke-WebRequest -Uri 'https://civitai.com/api/download/models/72591' -OutFile 'models/soulcard.safetensors' - - name: Run main app + - name: Run text2image app run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" - "${{ env.build_dir }}/samples/cpp/text2image/Release/stable_diffusion.exe ./models/dreamlike-art-dreamlike-anime-1.0/FP16 'cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting'" + "${{ env.build_dir }}/samples/cpp/image_generation/Release/text2image.exe ./models/dreamlike-art-dreamlike-anime-1.0/FP16 'cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting'" env: PATH: ${{ env.build_dir }}\openvino_genai - - name: Run LoRA app + - name: Run lora_text2image app run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" - "${{ env.build_dir }}/samples/cpp/text2image/Release/lora_stable_diffusion.exe ./models/dreamlike-art-dreamlike-anime-1.0/FP16 'curly-haired unicorn in the forest, anime, line' ./models/soulcard.safetensors 0.7" + "${{ env.build_dir }}/samples/cpp/image_generation/Release/lora_text2image.exe ./models/dreamlike-art-dreamlike-anime-1.0/FP16 'curly-haired unicorn in the forest, anime, line' ./models/soulcard.safetensors 0.7" env: PATH: ${{ env.build_dir }}\openvino_genai - - name: Run Python main app + - name: Run text2image.py app run: | . "./openvino_sd_cpp/Scripts/Activate.ps1" . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" $env:Path += "${{ env.build_dir }}\openvino_genai" - python .\samples\python\text2image\main.py .\models\dreamlike-art-dreamlike-anime-1.0\FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" + python .\samples\python\image_generation\text2image.py .\models\dreamlike-art-dreamlike-anime-1.0\FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" env: PYTHONPATH: ${{ env.build_dir }} - - name: Run Python LoRA app + - name: Run lora_text2image.py app run: | . "./openvino_sd_cpp/Scripts/Activate.ps1" . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" $env:Path += "${{ env.build_dir }}\openvino_genai" - python .\samples\python\text2image\lora.py .\models\dreamlike-art-dreamlike-anime-1.0\FP16 "curly-haired unicorn in the forest, anime, line" .\models\soulcard.safetensors 0.7 + python .\samples\python\image_generation\lora_text2image.py .\models\dreamlike-art-dreamlike-anime-1.0\FP16 "curly-haired unicorn in the forest, anime, line" .\models\soulcard.safetensors 0.7 env: PYTHONPATH: ${{ env.build_dir }} diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index f48e80413e..649d678c02 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -1,12 +1,12 @@ name: Windows (VS 2019, Python 3.11) -# on: -# workflow_dispatch: -# pull_request: -# merge_group: -# push: -# branches: -# - master -# - 'releases/**' +on: + workflow_dispatch: + pull_request: + merge_group: + push: + branches: + - master + - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions @@ -235,7 +235,7 @@ jobs: - name: Test bindings run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${env:OV_INSTALL_DIR}/wheels --upgrade-strategy eager + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${env:OV_INSTALL_DIR}/wheels python -m pytest -v ./tests/python_tests/test_chat_generate_api.py::test_set_chat_template env: PYTHONPATH: "./build/" # cmd evaluates variables in a different way. Setting PYTHONPATH before setupvars.bat instead of doing that after solves that. @@ -244,6 +244,7 @@ jobs: run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" python -m pip install . --verbose --find-links ${env:OV_INSTALL_DIR}/wheels + python -m pip install ./tools/who_what_benchmark --find-links ${env:OV_INSTALL_DIR}/wheels python -m pytest -v ./tests/python_tests/ --ignore ./tests/python_tests/test_whisper_generate_api.py --ignore ./tests/python_tests/test_vlm_api.py -k "not test_set_chat_template" genai_python_lib_whisper: @@ -299,7 +300,7 @@ jobs: - name: Test bindings run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${env:OV_INSTALL_DIR}/wheels --upgrade-strategy eager + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${env:OV_INSTALL_DIR}/wheels python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k test_smoke env: PYTHONPATH: "./build/" # cmd evaluates variables in a different way. Setting PYTHONPATH before setupvars.bat instead of doing that after solves that. @@ -308,6 +309,7 @@ jobs: run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" python -m pip install . --verbose --find-links ${env:OV_INSTALL_DIR}/wheels + python -m pip install ./tools/who_what_benchmark --find-links ${env:OV_INSTALL_DIR}/wheels python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k "not test_smoke" genai_python_lib_vlm: @@ -363,7 +365,7 @@ jobs: - name: Test bindings run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${env:OV_INSTALL_DIR}/wheels --upgrade-strategy eager + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${env:OV_INSTALL_DIR}/wheels python -m pytest -v ./tests/python_tests/test_vlm_api.py env: PYTHONPATH: "./build/" # cmd evaluates variables in a different way. Setting PYTHONPATH before setupvars.bat instead of doing that after solves that. @@ -425,7 +427,7 @@ jobs: run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --find-links ${env:OV_INSTALL_DIR}/wheels - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --find-links ${env:OV_INSTALL_DIR}/wheels + python -m pip install -r ./samples/requirements.txt --find-links ${env:OV_INSTALL_DIR}/wheels optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 optimum-cli export openvino --trust-remote-code --model openai/whisper-tiny whisper-tiny From b722d39f906ba46cf5e8ed612dcf26b59bd41d1a Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 11 Dec 2024 15:30:59 +0100 Subject: [PATCH 24/47] Add genai_nodejs_bindings linux workflow job --- .github/workflows/linux.yml | 81 ++++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 2916c340bf..09554c2c1b 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -483,9 +483,88 @@ jobs: source ${OV_INSTALL_DIR}/setupvars.sh timeout 25s ${OV_INSTALL_DIR}/samples/python/whisper_speech_recognition/whisper_speech_recognition.py ./whisper-tiny/ how_are_you_doing_today.wav + genai_nodejs_bindings: + name: Produce genai nodejs binaries archive + timeout-minutes: 150 + defaults: + run: + shell: bash + runs-on: ubuntu-20.04-16-cores + env: + DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input + CMAKE_BUILD_TYPE: 'Release' + CMAKE_GENERATOR: 'Ninja Multi-Config' + CMAKE_CXX_COMPILER_LAUNCHER: ccache + CMAKE_C_COMPILER_LAUNCHER: ccache + OPENVINO_REPO: ${{ github.workspace }}/openvino + GENAI_REPO: ${{ github.workspace }}/openvino.genai + BUILD_DIR: ${{ github.workspace }}/build + CCACHE_DIR: ${{ github.workspace }}/ccache + CCACHE_MAXSIZE: 2000Mi + + steps: + - name: Set apt + run: | + echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null + echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null + echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null + echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null + + - name: Clone OpenVINO + uses: actions/checkout@v4 + with: + repository: 'openvinotoolkit/openvino' + path: ${{ env.OPENVINO_REPO }} + submodules: 'true' + ref: ${{ env.OV_BRANCH}} + + - name: Install build dependencies + run: | + sudo -E ${OPENVINO_REPO}/install_build_dependencies.sh + sudo apt-get install ccache + + - name: Clone GenAI + uses: actions/checkout@v4 + with: + path: ${{ env.GENAI_REPO }} + submodules: recursive + + - name: Setup ccache + uses: actions/cache@v4 + with: + # Should save cache only if run in the master branch of the base repo + # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push + save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} + path: ${{ env.CCACHE_DIR }} + key: ${{ runner.os }}-${{ runner.arch }}-ccache-ov-and-genai-${{ matrix.build-type }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-ccache-ov-and-genai-${{ matrix.build-type }} + + - name: Build openvino + genai + run: | + cmake -DOPENVINO_EXTRA_MODULES=../openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ + -DCPACK_GENERATOR=NPM -DENABLE_JS=ON -UTBB* -DENABLE_SYSTEM_TBB=OFF \ + -DENABLE_PYTHON=OFF \ + -DENABLE_WHEEL=OFF \ + -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ + -S ./openvino -B ./build + cmake --build ./build --target package -j + + # + # Upload build artifacts and logs + # + + - name: Upload genai nodejs bindings archive + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: genai_nodejs_bindings + path: ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz + if-no-files-found: 'error' + Overall_Status: name: ci/gha_overall_status_linux - needs: [openvino_download, openvino_build, genai_python_lib, genai_package, genai_python_lib_whisper] + needs: [openvino_download, openvino_build, genai_python_lib, genai_package, genai_python_lib_whisper, genai_nodejs_bindings] if: ${{ always() }} runs-on: ubuntu-latest steps: From 3b4b3c0deee8036503ff1f673fdc925af4108a33 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Thu, 12 Dec 2024 10:09:49 +0100 Subject: [PATCH 25/47] Specify rpath for genai_node_addon.node --- src/cpp/CMakeLists.txt | 4 ++-- src/js/CMakeLists.txt | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt index ef719ff1be..2fcc37996a 100644 --- a/src/cpp/CMakeLists.txt +++ b/src/cpp/CMakeLists.txt @@ -141,10 +141,10 @@ if(CPACK_GENERATOR STREQUAL "NPM") # setting RPATH / LC_RPATH depending on platform if(LINUX) - # to find libopenvino_genai.so in the same folder + # to find libopenvino.so in the same folder set(rpaths "$ORIGIN") elseif(APPLE) - # to find libopenvino_genai.dylib in the same folder + # to find libopenvino.dylib in the same folder set(rpaths "@loader_path") endif() diff --git a/src/js/CMakeLists.txt b/src/js/CMakeLists.txt index 275803a804..d7ea811618 100644 --- a/src/js/CMakeLists.txt +++ b/src/js/CMakeLists.txt @@ -73,6 +73,19 @@ set_target_properties(${PROJECT_NAME} PROPERTIES SUFFIX ".node" ) +# setting RPATH / LC_RPATH depending on platform +if(LINUX) + # to find libopenvino_genai.so in the same folder + set(rpaths "$ORIGIN") +elseif(APPLE) + # to find libopenvino_genai.dylib in the same folder + set(rpaths "@loader_path") +endif() + +if(rpaths) + set_target_properties(${PROJECT_NAME} PROPERTIES INSTALL_RPATH "${rpaths}") +endif() + install(TARGETS ${PROJECT_NAME} LIBRARY DESTINATION . COMPONENT ${PROJECT_NAME} RUNTIME DESTINATION . COMPONENT ${PROJECT_NAME} From 17f2069c711ce1b748b5eaf9802cb36f44612657 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Thu, 12 Dec 2024 10:13:43 +0100 Subject: [PATCH 26/47] Fix offsets --- src/cpp/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt index 2fcc37996a..7864158e06 100644 --- a/src/cpp/CMakeLists.txt +++ b/src/cpp/CMakeLists.txt @@ -177,9 +177,9 @@ install(EXPORT OpenVINOGenAITargets FILE OpenVINOGenAITargets.cmake include(CMakePackageConfigHelpers) configure_package_config_file("${OpenVINOGenAI_SOURCE_DIR}/cmake/templates/OpenVINOGenAIConfig.cmake.in" - "${CMAKE_BINARY_DIR}/OpenVINOGenAIConfig.cmake" INSTALL_DESTINATION runtime/cmake) + "${CMAKE_BINARY_DIR}/OpenVINOGenAIConfig.cmake" INSTALL_DESTINATION runtime/cmake) write_basic_package_version_file("${CMAKE_BINARY_DIR}/OpenVINOGenAIConfigVersion.cmake" - VERSION ${OpenVINOGenAI_VERSION} COMPATIBILITY AnyNewerVersion) + VERSION ${OpenVINOGenAI_VERSION} COMPATIBILITY AnyNewerVersion) install(FILES "${CMAKE_BINARY_DIR}/OpenVINOGenAIConfig.cmake" "${CMAKE_BINARY_DIR}/OpenVINOGenAIConfigVersion.cmake" DESTINATION runtime/cmake COMPONENT core_genai_dev) export(EXPORT OpenVINOGenAITargets FILE "${CMAKE_BINARY_DIR}/OpenVINOGenAITargets.cmake" NAMESPACE openvino::) From 27a97d81de0a96ce99d5f6cc4fb2b9ecd1a8f0b6 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Thu, 12 Dec 2024 10:16:50 +0100 Subject: [PATCH 27/47] Fix path to js samples in readme Co-authored-by: Vladimir Zlobin --- src/js/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/js/README.md b/src/js/README.md index 77ebc0792c..19023eb3e9 100644 --- a/src/js/README.md +++ b/src/js/README.md @@ -19,7 +19,7 @@ TODO: Add instructions - To run sample you should have prepared model. Use this instruction [to download model](https://github.com/openvinotoolkit/openvino.genai/blob/master/samples/cpp/chat_sample/README.md#download-and-convert-the-model-and-tokenizers) -- Go to [samples](../../samples/js/) +- Go to [samples/js](../../samples/js/) - Run `node app.js`, you should see: `User Prompt: ...` ### Using as npm Dependency From 6f0e0b45039627f202efbdb22617053404eb2d89 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Thu, 12 Dec 2024 13:47:07 +0100 Subject: [PATCH 28/47] Align API --- samples/js/interactive.js | 56 ------------------------------------- src/js/lib/module.js | 20 ++++++------- src/js/tests/module.test.js | 24 ++++++++-------- 3 files changed, 22 insertions(+), 78 deletions(-) delete mode 100644 samples/js/interactive.js diff --git a/samples/js/interactive.js b/samples/js/interactive.js deleted file mode 100644 index 466326ced0..0000000000 --- a/samples/js/interactive.js +++ /dev/null @@ -1,56 +0,0 @@ -import path from 'node:path'; -import readline from 'readline'; - -import { Pipeline } from 'genai-node'; - -const MODEL_PATH = process.argv[2]; - -if (!MODEL_PATH) { - console.error('Please specify path to model directory\n' - + 'Run command must be: `node app.js *path_to_model_dir*`'); - process.exit(1); -} - -main(); - -async function main() { - // Create interface for reading user input from stdin - const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout, - }); - - console.log(`Welcome! Model "${path.parse(MODEL_PATH).name}" loaded. ` - + 'Type something and press enter. Type "finish" to exit.'); - const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH); - await pipeline.startChat(); - promptUser(); - - // Function to prompt the user for input - function promptUser() { - rl.question('> ', handleInput); - } - - // Function to handle user input - async function handleInput(input) { - input = input.trim(); - - // Check for exit command - if (input === 'finish') { - console.log('Goodbye!'); - await pipeline.finishChat(); - rl.close(); - process.exit(0); - } - - const result = await pipeline.generate(input, generationCallback); - console.log('\n'); - - // Wait for new input - promptUser(); - } - - function generationCallback(chunk) { - process.stdout.write(chunk); - } -} diff --git a/src/js/lib/module.js b/src/js/lib/module.js index 2f3d557711..b390712fa4 100644 --- a/src/js/lib/module.js +++ b/src/js/lib/module.js @@ -55,7 +55,14 @@ class LLMPipeline { return result; } - async generate(prompt, generationCallback, options = {}) { + async generate(prompt, generationCallbackOrOptions, generationCallback) { + let options = {}; + + if (!generationCallback) + generationCallback = generationCallbackOrOptions; + else + options = generationCallbackOrOptions; + if (!this.isInitialized) throw new Error('Pipeline is not initialized'); @@ -95,24 +102,17 @@ class LLMPipeline { } } -const availablePipelines = { LLMPipeline: LLMPipeline }; - class Pipeline { - static async create(pipelineType, modelPath, device = 'CPU') { - if (!Object.keys(availablePipelines).includes(pipelineType)) - throw new Error(`Pipeline type: '${pipelineType}' doesn't support`); - - const pipeline = new availablePipelines[pipelineType](modelPath, device); + static async LLMPipeline(modelPath, device = 'CPU') { + const pipeline = new LLMPipeline(modelPath, device); await pipeline.init(); return pipeline; } } -const availablePipelinesKeys = Object.keys(availablePipelines); export { addon, Pipeline, - availablePipelinesKeys as availablePipelines, }; diff --git a/src/js/tests/module.test.js b/src/js/tests/module.test.js index 90b2f5266d..672fa71638 100644 --- a/src/js/tests/module.test.js +++ b/src/js/tests/module.test.js @@ -11,7 +11,7 @@ describe('module', async () => { let pipeline = null; await before(async () => { - pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); await pipeline.startChat(); }); @@ -23,8 +23,8 @@ describe('module', async () => { await it('should generate "Hello world"', async () => { const result = await pipeline.generate( 'Type "Hello world!" in English', + { temperature: '0', max_new_tokens: '4' }, () => {}, - { temperature: '0', max_new_tokens: '4' } ); assert.strictEqual(result, 'Hello world!'); @@ -33,7 +33,7 @@ describe('module', async () => { describe('corner cases', async () => { it('should throw an error if pipeline is already initialized', async () => { - const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); await assert.rejects( async () => await pipeline.init(), @@ -45,7 +45,7 @@ describe('corner cases', async () => { }); it('should throw an error if chat is already started', async () => { - const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); await pipeline.startChat(); @@ -59,7 +59,7 @@ describe('corner cases', async () => { }); it('should throw an error if chat is not started', async () => { - const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); await assert.rejects( () => pipeline.finishChat(), @@ -75,7 +75,7 @@ describe('generation parameters validation', () => { let pipeline = null; before(async () => { - pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); await pipeline.startChat(); }); @@ -95,7 +95,7 @@ describe('generation parameters validation', () => { }); it('should throw an error if generationCallback is not a function', async () => { - const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); await pipeline.startChat(); @@ -110,7 +110,7 @@ describe('generation parameters validation', () => { it('should throw an error if options specified but not an object', async () => { await assert.rejects( - async () => await pipeline.generate('prompt', () => {}, 'options'), + async () => await pipeline.generate('prompt', 'options', () => {}), { name: 'Error', message: 'Options must be an object', @@ -120,7 +120,7 @@ describe('generation parameters validation', () => { it('should perform generation with default options', async () => { try { - await pipeline.generate('prompt', () => {}, { max_new_tokens: 1 }); + await pipeline.generate('prompt', { max_new_tokens: 1 }, () => {}); } catch (error) { assert.fail(error); } @@ -129,14 +129,14 @@ describe('generation parameters validation', () => { }); it('should return a string as generation result', async () => { - const reply = await pipeline.generate('prompt', () => {}, { max_new_tokens: 1 }); + const reply = await pipeline.generate('prompt', { max_new_tokens: 1 }, () => {}); assert.strictEqual(typeof reply, 'string'); }); it('should call generationCallback with string chunk', async () => { - await pipeline.generate('prompt', (chunk) => { + await pipeline.generate('prompt', { max_new_tokens: 1 }, (chunk) => { assert.strictEqual(typeof chunk, 'string'); - }, { max_new_tokens: 1 }); + }); }); }); From 855c893f5e84a93e6c2c1bf47dcddecccbb31a37 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Thu, 12 Dec 2024 13:47:59 +0100 Subject: [PATCH 29/47] Align chat sample --- samples/js/app.js | 28 ---------- samples/js/{ => chat_sample}/.gitignore | 0 samples/js/chat_sample/README.md | 0 samples/js/chat_sample/chat_sample.js | 54 +++++++++++++++++++ .../js/{ => chat_sample}/package-lock.json | 11 ++-- samples/js/{ => chat_sample}/package.json | 2 +- 6 files changed, 63 insertions(+), 32 deletions(-) delete mode 100644 samples/js/app.js rename samples/js/{ => chat_sample}/.gitignore (100%) create mode 100644 samples/js/chat_sample/README.md create mode 100644 samples/js/chat_sample/chat_sample.js rename samples/js/{ => chat_sample}/package-lock.json (71%) rename samples/js/{ => chat_sample}/package.json (82%) diff --git a/samples/js/app.js b/samples/js/app.js deleted file mode 100644 index f57e2fad1d..0000000000 --- a/samples/js/app.js +++ /dev/null @@ -1,28 +0,0 @@ -import { Pipeline } from 'genai-node'; - -const MODEL_PATH = process.argv[2]; - -if (!MODEL_PATH) { - console.error('Please specify path to model directory\n' - + 'Run command must be: `node app.js *path_to_model_dir*`'); - process.exit(1); -} - -const generationCallback = (chunk) => { - process.stdout.write(chunk); -}; - -const prompt = 'Who are you?'; -console.log(`User Prompt: "${prompt}"\n`); - -const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH); - -await pipeline.startChat(); -const result = await pipeline.generate( - prompt, - generationCallback, - { temperature: 0 }, -); -await pipeline.finishChat(); - -console.log(`\n\nGeneration result:\n"${result}"`) diff --git a/samples/js/.gitignore b/samples/js/chat_sample/.gitignore similarity index 100% rename from samples/js/.gitignore rename to samples/js/chat_sample/.gitignore diff --git a/samples/js/chat_sample/README.md b/samples/js/chat_sample/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/js/chat_sample/chat_sample.js b/samples/js/chat_sample/chat_sample.js new file mode 100644 index 0000000000..fad1b507a8 --- /dev/null +++ b/samples/js/chat_sample/chat_sample.js @@ -0,0 +1,54 @@ +import readline from 'readline'; +import { Pipeline } from 'genai-node'; + +main(); + +function streamer(subword) { + process.stdout.write(subword); +} + +async function main() { + const MODEL_PATH = process.argv[2]; + + if (!MODEL_PATH) { + console.error('Please specify path to model directory\n' + + 'Run command must be: `node app.js *path_to_model_dir*`'); + process.exit(1); + } + + const device = 'CPU'; // GPU can be used as well + + // Create interface for reading user input from stdin + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + + const pipe = await Pipeline.LLMPipeline(MODEL_PATH, device); + const config = { 'max_new_tokens': 100 }; + + await pipe.startChat(); + promptUser(); + + // Function to prompt the user for input + function promptUser() { + rl.question('question:\n', handleInput); + } + + // Function to handle user input + async function handleInput(input) { + input = input.trim(); + + // Check for exit command + if (!input) { + await pipe.finishChat(); + rl.close(); + process.exit(0); + } + + await pipe.generate(input, config, streamer); + console.log('\n----------'); + + promptUser(); + } +} diff --git a/samples/js/package-lock.json b/samples/js/chat_sample/package-lock.json similarity index 71% rename from samples/js/package-lock.json rename to samples/js/chat_sample/package-lock.json index 4b2d14913f..fbee0db012 100644 --- a/samples/js/package-lock.json +++ b/samples/js/chat_sample/package-lock.json @@ -9,13 +9,13 @@ "version": "1.0.0", "license": "Apache-2.0", "devDependencies": { - "genai-node": "../../src/js/" + "genai-node": "../../../src/js/" }, "engines": { "node": ">=21.0.0" } }, - "../../src/js": { + "../../../src/js": { "name": "genai-node", "version": "2024.5.0-preview", "dev": true, @@ -25,12 +25,17 @@ "darwin", "win32" ], + "devDependencies": { + "@huggingface/hub": "^0.21.0", + "global-agent": "^3.0.0", + "node-fetch": "^3.3.2" + }, "engines": { "node": ">=21.0.0" } }, "node_modules/genai-node": { - "resolved": "../../src/js", + "resolved": "../../../src/js", "link": true } } diff --git a/samples/js/package.json b/samples/js/chat_sample/package.json similarity index 82% rename from samples/js/package.json rename to samples/js/chat_sample/package.json index 0251bdfa14..8de12582e9 100644 --- a/samples/js/package.json +++ b/samples/js/chat_sample/package.json @@ -4,7 +4,7 @@ "license": "Apache-2.0", "type": "module", "devDependencies": { - "genai-node": "../../src/js/" + "genai-node": "../../../src/js/" }, "engines": { "node": ">=21.0.0" From df51af9f9a189c3d413bf253db03e9c7271fc9e1 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Thu, 12 Dec 2024 14:23:55 +0100 Subject: [PATCH 30/47] Add README.md into js chat sample --- samples/js/chat_sample/README.md | 49 ++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/samples/js/chat_sample/README.md b/samples/js/chat_sample/README.md index e69de29bb2..8381c98aa8 100644 --- a/samples/js/chat_sample/README.md +++ b/samples/js/chat_sample/README.md @@ -0,0 +1,49 @@ +# JavaScript chat_sample that supports most popular models like LLaMA 3 + +This example showcases inference of text-generation Large Language Models (LLMs): `chatglm`, `LLaMA`, `Qwen` and other models with the same signature. The application doesn't have many configuration options to encourage the reader to explore and modify the source code. For example, change the device for inference to GPU. The sample fearures `Pipeline.LLMPipeline` and configures it for the chat scenario. + +## Download and convert the model and tokenizers + +To convert model you have to use python package `optimum-intel`. +The `--upgrade-strategy eager` option is needed to ensure `optimum-intel` is upgraded to the latest version. + +Install [../../export-requirements.txt](../../export-requirements.txt) to convert a model. + +```sh +pip install --upgrade-strategy eager -r ../../export-requirements.txt +optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 +``` + +## Run: + +Create `bin` folder in [../../../src/js](../../../src/js). +Compile GenAI JavaScript bindings archive first. Put its content into `bin` directory. + +Run `npm install` in current folder and then run a sample: + +`node chat_sample.js TinyLlama-1.1B-Chat-v1.0` + +Discrete GPUs (dGPUs) usually provide better performance compared to CPUs. It is recommended to run larger models on a dGPU with 32GB+ RAM. For example, the model meta-llama/Llama-2-13b-chat-hf can benefit from being run on a dGPU. Modify the source code to change the device for inference to the GPU. + +See https://github.com/openvinotoolkit/openvino.genai/blob/master/src/README.md#supported-models for the list of supported models. + +### Troubleshooting + +#### Unicode characters encoding error on Windows + +Example error: +``` +UnicodeEncodeError: 'charmap' codec can't encode character '\u25aa' in position 0: character maps to +``` + +If you encounter the error described in the example when sample is printing output to the Windows console, it is likely due to the default Windows encoding not supporting certain Unicode characters. To resolve this: +1. Enable Unicode characters for Windows cmd - open `Region` settings from `Control panel`. `Administrative`->`Change system locale`->`Beta: Use Unicode UTF-8 for worldwide language support`->`OK`. Reboot. +2. Enable UTF-8 mode by setting environment variable `PYTHONIOENCODING="utf8"`. + +#### Missing chat template + +If you encounter an exception indicating a missing "chat template" when launching the `ov::genai::LLMPipeline` in chat mode, it likely means the model was not tuned for chat functionality. To work this around, manually add the chat template to tokenizer_config.json of your model. +The following template can be used as a default, but it may not work properly with every model: +``` +"chat_template": "{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|im_start|>user\n' + message['content'] + '<|im_end|>\n<|im_start|>assistant\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|im_end|>\n'}}{% endif %}{% endfor %}", +``` From a365ef8c9f90a981b38624f875c6cd1e2ebc1858 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Thu, 12 Dec 2024 15:53:50 +0100 Subject: [PATCH 31/47] Update build instructions --- src/js/README.md | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/src/js/README.md b/src/js/README.md index 19023eb3e9..2302ce1082 100644 --- a/src/js/README.md +++ b/src/js/README.md @@ -13,14 +13,40 @@ This is preview version, do not use it in production! ### Build Bindings -TODO: Add instructions +#### Build OpenVINO GenAI as OpenVINO Extra Module + +OpenVINO GenAI Node.js bindings can be built as an extra module during the OpenVINO build process. This method simplifies the build process by integrating OpenVINO GenAI directly into the OpenVINO build. + +1. Clone OpenVINO repository: + ```sh + git clone --recursive https://github.com/openvinotoolkit/openvino.git + ``` +1. Configure CMake with OpenVINO extra modules: + ```sh + cmake -DOPENVINO_EXTRA_MODULES=*relative (from openvino folder) path to genai repository* -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ + -DCPACK_GENERATOR=NPM -DENABLE_JS=ON -UTBB* -DENABLE_SYSTEM_TBB=OFF \ + -DENABLE_PYTHON=OFF \ + -DENABLE_WHEEL=OFF \ + -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ + -S ./openvino -B ./build + ``` +1. Build OpenVINO archive with GenAI: + ```sh + cmake --build ./build --target package -j + ``` + +1. In `build` folder you will find `genai_nodejs_bindings.tar.gz`. + Create `bin` directory by path `src/js/` and unarchive archive content to it. +1. Run tests to be sure that everything works: + `npm test` ### Perform Test Run - To run sample you should have prepared model. - Use this instruction [to download model](https://github.com/openvinotoolkit/openvino.genai/blob/master/samples/cpp/chat_sample/README.md#download-and-convert-the-model-and-tokenizers) -- Go to [samples/js](../../samples/js/) -- Run `node app.js`, you should see: `User Prompt: ...` + Use this instruction [to download model](https://github.com/openvinotoolkit/openvino.genai/blob/master/samples/js/chat_sample/README.md#download-and-convert-the-model-and-tokenizers) +- Go to [samples/js/chat_sample/](../../samples/js/chat_sample/) +- Read [README.md](../../samples/js/chat_sample/README.md) and follow steps there + to run **chat sample**. ### Using as npm Dependency From 6d0b501064f19e2e72e4c413d1073cc73c0ab9e9 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 16 Dec 2024 15:16:00 +0100 Subject: [PATCH 32/47] Fix review comments --- src/js/CMakeLists.txt | 3 ++- src/js/README.md | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/js/CMakeLists.txt b/src/js/CMakeLists.txt index d7ea811618..7e4ff0bea4 100644 --- a/src/js/CMakeLists.txt +++ b/src/js/CMakeLists.txt @@ -17,7 +17,8 @@ endif() project(genai_node_addon) -# Add definitions +# Specify NAPI version 8 +# supports v12.22.0+, v14.17.0+, v15.12.0+, 16.0.0 and all later Node.js versions add_definitions(-DNAPI_VERSION=8) include(FetchContent) diff --git a/src/js/README.md b/src/js/README.md index 2302ce1082..9092340d83 100644 --- a/src/js/README.md +++ b/src/js/README.md @@ -23,7 +23,7 @@ OpenVINO GenAI Node.js bindings can be built as an extra module during the OpenV ``` 1. Configure CMake with OpenVINO extra modules: ```sh - cmake -DOPENVINO_EXTRA_MODULES=*relative (from openvino folder) path to genai repository* -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ + cmake -DOPENVINO_EXTRA_MODULES=*absolute path to genai repository directory* -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ -DCPACK_GENERATOR=NPM -DENABLE_JS=ON -UTBB* -DENABLE_SYSTEM_TBB=OFF \ -DENABLE_PYTHON=OFF \ -DENABLE_WHEEL=OFF \ @@ -36,7 +36,10 @@ OpenVINO GenAI Node.js bindings can be built as an extra module during the OpenV ``` 1. In `build` folder you will find `genai_nodejs_bindings.tar.gz`. - Create `bin` directory by path `src/js/` and unarchive archive content to it. + Create `bin` directory by path `src/js/` and unpack archive content to it. + ```sh + tar -xzf genai_nodejs_bindings.tar.gz + ``` 1. Run tests to be sure that everything works: `npm test` From f241cda3ffaeeb03be638249809affb94adea9a2 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 16 Dec 2024 16:13:10 +0100 Subject: [PATCH 33/47] Add run js api test in linux workflow --- .github/workflows/linux.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 1d8cdb0385..7a7e9fc130 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -469,6 +469,15 @@ jobs: -S ./openvino -B ./build cmake --build ./build --target package -j + - name: Run javascript tests + working-directory: ${{ env.GENAI_REPO }}/src/js + run: | + mkdir bin + cp ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz bin + cd bin && tar -xzf genai_nodejs_bindings.tar.gz && rm genai_nodejs_bindings.tar.gz cd .. + npm install + npm test + # # Upload build artifacts and logs # From 1f2324b23e01222fb952218c8eb99605813ab4f9 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 16 Dec 2024 16:52:54 +0100 Subject: [PATCH 34/47] Fix workflow --- .github/workflows/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 8487c7cf4e..bb448a0e49 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -470,7 +470,7 @@ jobs: run: | mkdir bin cp ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz bin - cd bin && tar -xzf genai_nodejs_bindings.tar.gz && rm genai_nodejs_bindings.tar.gz cd .. + cd bin && tar -xzf genai_nodejs_bindings.tar.gz && rm genai_nodejs_bindings.tar.gz && cd .. npm install npm test From dda494de1bd0f1351dd55882576cf25f2a92e90e Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 17 Dec 2024 19:29:14 +0100 Subject: [PATCH 35/47] Add test for js sample --- samples/js/chat_sample/chat_sample.js | 2 +- samples/js/chat_sample/package.json | 3 ++ samples/js/chat_sample/tests/usage.test.js | 57 ++++++++++++++++++++++ samples/js/interactive.js | 56 +++++++++++++++++++++ 4 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 samples/js/chat_sample/tests/usage.test.js create mode 100644 samples/js/interactive.js diff --git a/samples/js/chat_sample/chat_sample.js b/samples/js/chat_sample/chat_sample.js index fad1b507a8..9bca37d4b7 100644 --- a/samples/js/chat_sample/chat_sample.js +++ b/samples/js/chat_sample/chat_sample.js @@ -49,6 +49,6 @@ async function main() { await pipe.generate(input, config, streamer); console.log('\n----------'); - promptUser(); + if (!rl.closed) promptUser(); } } diff --git a/samples/js/chat_sample/package.json b/samples/js/chat_sample/package.json index 8de12582e9..24e66a120d 100644 --- a/samples/js/chat_sample/package.json +++ b/samples/js/chat_sample/package.json @@ -8,5 +8,8 @@ }, "engines": { "node": ">=21.0.0" + }, + "scripts": { + "test": "node tests/usage.test.js" } } diff --git a/samples/js/chat_sample/tests/usage.test.js b/samples/js/chat_sample/tests/usage.test.js new file mode 100644 index 0000000000..639ed0a487 --- /dev/null +++ b/samples/js/chat_sample/tests/usage.test.js @@ -0,0 +1,57 @@ +import { env } from 'process'; +import { spawn } from 'child_process'; + +const MODEL_PATH = env.MODEL_PATH; + +if (!MODEL_PATH) + throw new Error( + 'Please environment variable MODEL_PATH to the path of the model directory' + ); + +const runTest = async () => { + return new Promise((resolve, reject) => { + const script = spawn('node', ['chat_sample.js', MODEL_PATH]); + let output = ''; + + // Collect output from stdout + script.stdout.on('data', (data) => { + output += data.toString(); + }); + + // Capture errors + script.stderr.on('data', (data) => { + reject(data.toString()); + }); + + // Send input after detecting the question prompt + script.stdout.once('data', (data) => { + if (data.toString().startsWith('question:')) { + script.stdin.write('Say exactly, without any changes, print it as is: "Hello world"\n'); // Provide input + script.stdin.end(); // Close stdin to signal EOF + } + }); + + // Check results when the process exits + script.on('close', (code) => { + if (code !== 0) { + return reject(`Process exited with code ${code}`); + } + // Validate the output + if (output.includes('"Hello world"')) { + resolve('Test passed!'); + } else { + reject('Test failed: Output did not match expected result.'); + } + }); + }); +}; + +runTest() + .then((message) => { + console.log(message); + process.exit(0); + }) + .catch((err) => { + console.error(err); + process.exit(1); + }); diff --git a/samples/js/interactive.js b/samples/js/interactive.js new file mode 100644 index 0000000000..466326ced0 --- /dev/null +++ b/samples/js/interactive.js @@ -0,0 +1,56 @@ +import path from 'node:path'; +import readline from 'readline'; + +import { Pipeline } from 'genai-node'; + +const MODEL_PATH = process.argv[2]; + +if (!MODEL_PATH) { + console.error('Please specify path to model directory\n' + + 'Run command must be: `node app.js *path_to_model_dir*`'); + process.exit(1); +} + +main(); + +async function main() { + // Create interface for reading user input from stdin + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + + console.log(`Welcome! Model "${path.parse(MODEL_PATH).name}" loaded. ` + + 'Type something and press enter. Type "finish" to exit.'); + const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH); + await pipeline.startChat(); + promptUser(); + + // Function to prompt the user for input + function promptUser() { + rl.question('> ', handleInput); + } + + // Function to handle user input + async function handleInput(input) { + input = input.trim(); + + // Check for exit command + if (input === 'finish') { + console.log('Goodbye!'); + await pipeline.finishChat(); + rl.close(); + process.exit(0); + } + + const result = await pipeline.generate(input, generationCallback); + console.log('\n'); + + // Wait for new input + promptUser(); + } + + function generationCallback(chunk) { + process.stdout.write(chunk); + } +} From 394d4250cf423eba3a4e5fdf2c0f1fd5869dd48f Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 17 Dec 2024 19:29:46 +0100 Subject: [PATCH 36/47] Extend linux workflow to run js samples test --- .github/workflows/linux.yml | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index bb448a0e49..3e5952b62c 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -486,6 +486,60 @@ jobs: path: ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz if-no-files-found: 'error' + genai_nodejs_samples_tests: + name: NodeJS Samples Tests + needs: [ genai_nodejs_bindings ] + timeout-minutes: 30 + defaults: + run: + shell: bash + runs-on: aks-linux-2-cores-8gb + env: + INSTALL_DIR: ${{ github.workspace }}/ov + SRC_DIR: ${{ github.workspace }}/src + MODELS_DIR: ${{ github.workspace }}/models + JS_SRC_DIR: ${{ github.workspace }}/src/openvino.genai/src/js + JS_SAMPLES_DIR: ${{ github.workspace }}/src/openvino.genai/samples/js + BIN_DIR: ${{ github.workspace }}/src/openvino.genai/src/js/bin + steps: + - name: Clone openvino.genai + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: ${{ env.SRC_DIR }} + submodules: recursive + + - name: Download genai nodejs bindings archive + uses: actions/download-artifact@v4 + with: + name: genai_nodejs_bindings + path: ${{ env.BIN_DIR }} + + - name: Unpack genai nodejs bindings archive + run: tar -xzf genai_nodejs_bindings.tar.gz && rm genai_nodejs_bindings.tar.gz + working-directory: ${{ env.BIN_DIR }} + + - name: Install nodejs dependencies + run: npm install + working-directory: ${{ env.JS_SRC_DIR }} + + - name: Download model for tests + run: npm run test_setup + working-directory: ${{ env.JS_SAMPLES_DIR }} + + - name: Link genai-node package + run: npm link + working-directory: ${{ env.JS_SRC_DIR }} + + - name: Install genai-node package to samples + run: npm link genai-node + working-directory: ${{ env.JS_SAMPLES_DIR }} + + - name: Run tests + run: npm test + env: + MODEL_PATH: ${{ env.JS_SRC_DIR }}/tests/models/Llama-3.2-3B-Instruct-openvino-8bit + working-directory: ${{ env.JS_SAMPLES_DIR }} + Overall_Status: name: ci/gha_overall_status_linux needs: [openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples, genai_tests_wheel, genai_samples_tests, genai_nodejs_bindings] From 31861422aba147a6fc47617353ecafa8ae860e09 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 17 Dec 2024 20:17:56 +0100 Subject: [PATCH 37/47] Change runner for js samples tests --- .github/workflows/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 3e5952b62c..9c89377bec 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -493,7 +493,7 @@ jobs: defaults: run: shell: bash - runs-on: aks-linux-2-cores-8gb + runs-on: ubuntu-20.04-16-cores env: INSTALL_DIR: ${{ github.workspace }}/ov SRC_DIR: ${{ github.workspace }}/src From 2fbd888f7940ebdb2592f2a6426fef387d08a896 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 17 Dec 2024 20:37:54 +0100 Subject: [PATCH 38/47] Fix repo path --- .github/workflows/linux.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 9c89377bec..df5eb4ca05 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -496,16 +496,16 @@ jobs: runs-on: ubuntu-20.04-16-cores env: INSTALL_DIR: ${{ github.workspace }}/ov - SRC_DIR: ${{ github.workspace }}/src + GENAI_REPO: ${{ github.workspace }}/openvino.genai MODELS_DIR: ${{ github.workspace }}/models - JS_SRC_DIR: ${{ github.workspace }}/src/openvino.genai/src/js - JS_SAMPLES_DIR: ${{ github.workspace }}/src/openvino.genai/samples/js - BIN_DIR: ${{ github.workspace }}/src/openvino.genai/src/js/bin + JS_SRC_DIR: ${{ github.workspace }}/openvino.genai/src/js + JS_SAMPLES_DIR: ${{ github.workspace }}/openvino.genai/samples/js + BIN_DIR: ${{ github.workspace }}/openvino.genai/src/js/bin steps: - name: Clone openvino.genai uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - path: ${{ env.SRC_DIR }} + path: ${{ env.GENAI_REPO }} submodules: recursive - name: Download genai nodejs bindings archive From 0a0c39995fdee5e5fa7d2eee2d69a60164f46e14 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 17 Dec 2024 21:30:23 +0100 Subject: [PATCH 39/47] Fix path --- .github/workflows/linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index fd3cd08547..44a659f132 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -499,7 +499,7 @@ jobs: GENAI_REPO: ${{ github.workspace }}/openvino.genai MODELS_DIR: ${{ github.workspace }}/models JS_SRC_DIR: ${{ github.workspace }}/openvino.genai/src/js - JS_SAMPLES_DIR: ${{ github.workspace }}/openvino.genai/samples/js + JS_SAMPLES_DIR: ${{ github.workspace }}/openvino.genai/samples/js/chat_sample BIN_DIR: ${{ github.workspace }}/openvino.genai/src/js/bin steps: - name: Clone openvino.genai @@ -524,7 +524,7 @@ jobs: - name: Download model for tests run: npm run test_setup - working-directory: ${{ env.JS_SAMPLES_DIR }} + working-directory: ${{ env.JS_SRC_DIR }} - name: Link genai-node package run: npm link From 35c1c61f7fae0aceaa384aa5077a01e673996fc3 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 17 Dec 2024 22:02:10 +0100 Subject: [PATCH 40/47] Add debug output --- samples/js/chat_sample/tests/usage.test.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/samples/js/chat_sample/tests/usage.test.js b/samples/js/chat_sample/tests/usage.test.js index 639ed0a487..8b6b8a3338 100644 --- a/samples/js/chat_sample/tests/usage.test.js +++ b/samples/js/chat_sample/tests/usage.test.js @@ -36,6 +36,9 @@ const runTest = async () => { if (code !== 0) { return reject(`Process exited with code ${code}`); } + + console.log(`Result output: ${output}`); + // Validate the output if (output.includes('"Hello world"')) { resolve('Test passed!'); From b2cb5dd3fc8ede15736db55f05c81e4c5e2fb4ff Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 18 Dec 2024 09:05:29 +0100 Subject: [PATCH 41/47] Fix js samples test --- samples/js/chat_sample/tests/usage.test.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/samples/js/chat_sample/tests/usage.test.js b/samples/js/chat_sample/tests/usage.test.js index 8b6b8a3338..63856cbe43 100644 --- a/samples/js/chat_sample/tests/usage.test.js +++ b/samples/js/chat_sample/tests/usage.test.js @@ -2,6 +2,8 @@ import { env } from 'process'; import { spawn } from 'child_process'; const MODEL_PATH = env.MODEL_PATH; +const prompt = 'Tell me exactly, no changes, print as is: "Hello world"'; +const expected = 'Hello world'; if (!MODEL_PATH) throw new Error( @@ -26,7 +28,7 @@ const runTest = async () => { // Send input after detecting the question prompt script.stdout.once('data', (data) => { if (data.toString().startsWith('question:')) { - script.stdin.write('Say exactly, without any changes, print it as is: "Hello world"\n'); // Provide input + script.stdin.write(`${prompt}\n`); // Provide input script.stdin.end(); // Close stdin to signal EOF } }); @@ -37,10 +39,11 @@ const runTest = async () => { return reject(`Process exited with code ${code}`); } + // Log the output console.log(`Result output: ${output}`); // Validate the output - if (output.includes('"Hello world"')) { + if (output.includes(expected)) { resolve('Test passed!'); } else { reject('Test failed: Output did not match expected result.'); From 30803b451fda376d912fa2bc0d92371411ebb98f Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 23 Dec 2024 11:08:38 +0100 Subject: [PATCH 42/47] Fix run sample cmd Co-authored-by: Vladimir Zlobin --- samples/js/chat_sample/chat_sample.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/js/chat_sample/chat_sample.js b/samples/js/chat_sample/chat_sample.js index 9bca37d4b7..cf4c5e7704 100644 --- a/samples/js/chat_sample/chat_sample.js +++ b/samples/js/chat_sample/chat_sample.js @@ -12,7 +12,7 @@ async function main() { if (!MODEL_PATH) { console.error('Please specify path to model directory\n' - + 'Run command must be: `node app.js *path_to_model_dir*`'); + + 'Run command must be: `node chat_sample.js *path_to_model_dir*`'); process.exit(1); } From b04ee6cbbdd51a129dcc06462a1ec12ba0ffdfcf Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 23 Dec 2024 11:14:07 +0100 Subject: [PATCH 43/47] Fix js README.md --- src/js/README.md | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/js/README.md b/src/js/README.md index 9092340d83..7971c238fc 100644 --- a/src/js/README.md +++ b/src/js/README.md @@ -1,4 +1,4 @@ -# OpenVINO™ GenAI Node.js bindings (preview) +we# OpenVINO™ GenAI Node.js bindings (preview) ## DISCLAIMER @@ -35,26 +35,22 @@ OpenVINO GenAI Node.js bindings can be built as an extra module during the OpenV cmake --build ./build --target package -j ``` -1. In `build` folder you will find `genai_nodejs_bindings.tar.gz`. - Create `bin` directory by path `src/js/` and unpack archive content to it. +1. Put Node.js bindings into npm package `bin` directory and install dependencies: ```sh - tar -xzf genai_nodejs_bindings.tar.gz + mkdir ./src/js/bin/ + tar -xvf ./build/genai_nodejs_bindings.tar.gz --directory ./src/js/bin/ + cd ./src/js/ + npm install ``` 1. Run tests to be sure that everything works: - `npm test` - -### Perform Test Run - -- To run sample you should have prepared model. - Use this instruction [to download model](https://github.com/openvinotoolkit/openvino.genai/blob/master/samples/js/chat_sample/README.md#download-and-convert-the-model-and-tokenizers) -- Go to [samples/js/chat_sample/](../../samples/js/chat_sample/) -- Read [README.md](../../samples/js/chat_sample/README.md) and follow steps there - to run **chat sample**. + ```sh + npm test + ``` ### Using as npm Dependency -To use this package locally use `npm link` in this directory -and `npm link genai-node` in the folder where you want add this package as dependency +To use this package locally use `npm link` in `src/js/` directory +and `npm link genai-node` in the folder where you want to add this package as a dependency To extract this package and use it as distributed npm package run `npm package`. This command creates archive that you may use in your projects. From d7e0d2b4c47f8e75c8d8c21e7d02f076c8cde1e4 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 23 Dec 2024 11:46:53 +0100 Subject: [PATCH 44/47] Replace AUTO device to CPU --- src/js/tests/bindings.test.js | 2 +- src/js/tests/module.test.js | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/js/tests/bindings.test.js b/src/js/tests/bindings.test.js index 3e852e5307..f76c45e0d9 100644 --- a/src/js/tests/bindings.test.js +++ b/src/js/tests/bindings.test.js @@ -13,7 +13,7 @@ describe('bindings', () => { before((_, done) => { pipeline = new addon.LLMPipeline(); - pipeline.init(MODEL_PATH, 'AUTO', (err) => { + pipeline.init(MODEL_PATH, 'CPU', (err) => { if (err) { console.error(err); process.exit(1); diff --git a/src/js/tests/module.test.js b/src/js/tests/module.test.js index 672fa71638..cbb9e905a4 100644 --- a/src/js/tests/module.test.js +++ b/src/js/tests/module.test.js @@ -11,7 +11,7 @@ describe('module', async () => { let pipeline = null; await before(async () => { - pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); + pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'CPU'); await pipeline.startChat(); }); @@ -33,7 +33,7 @@ describe('module', async () => { describe('corner cases', async () => { it('should throw an error if pipeline is already initialized', async () => { - const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); + const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'CPU'); await assert.rejects( async () => await pipeline.init(), @@ -45,7 +45,7 @@ describe('corner cases', async () => { }); it('should throw an error if chat is already started', async () => { - const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); + const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'CPU'); await pipeline.startChat(); @@ -59,7 +59,7 @@ describe('corner cases', async () => { }); it('should throw an error if chat is not started', async () => { - const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); + const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'CPU'); await assert.rejects( () => pipeline.finishChat(), @@ -75,7 +75,7 @@ describe('generation parameters validation', () => { let pipeline = null; before(async () => { - pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); + pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'CPU'); await pipeline.startChat(); }); @@ -95,7 +95,7 @@ describe('generation parameters validation', () => { }); it('should throw an error if generationCallback is not a function', async () => { - const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); + const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'CPU'); await pipeline.startChat(); From d1ce555b354937ed283ce92c6f1b82bc634a811b Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 23 Dec 2024 12:11:10 +0100 Subject: [PATCH 45/47] Fix js samples README --- samples/js/chat_sample/README.md | 5 ++- samples/js/interactive.js | 56 -------------------------------- 2 files changed, 2 insertions(+), 59 deletions(-) delete mode 100644 samples/js/interactive.js diff --git a/samples/js/chat_sample/README.md b/samples/js/chat_sample/README.md index 8381c98aa8..46caba48e3 100644 --- a/samples/js/chat_sample/README.md +++ b/samples/js/chat_sample/README.md @@ -16,10 +16,9 @@ optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B ## Run: -Create `bin` folder in [../../../src/js](../../../src/js). -Compile GenAI JavaScript bindings archive first. Put its content into `bin` directory. +Compile GenAI JavaScript bindings archive first using the instructions in [../../../src/js/README.md](../../../src/js/README.md#build-bindings). -Run `npm install` in current folder and then run a sample: +Run `npm install` in current folder and then run the sample: `node chat_sample.js TinyLlama-1.1B-Chat-v1.0` diff --git a/samples/js/interactive.js b/samples/js/interactive.js deleted file mode 100644 index 466326ced0..0000000000 --- a/samples/js/interactive.js +++ /dev/null @@ -1,56 +0,0 @@ -import path from 'node:path'; -import readline from 'readline'; - -import { Pipeline } from 'genai-node'; - -const MODEL_PATH = process.argv[2]; - -if (!MODEL_PATH) { - console.error('Please specify path to model directory\n' - + 'Run command must be: `node app.js *path_to_model_dir*`'); - process.exit(1); -} - -main(); - -async function main() { - // Create interface for reading user input from stdin - const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout, - }); - - console.log(`Welcome! Model "${path.parse(MODEL_PATH).name}" loaded. ` - + 'Type something and press enter. Type "finish" to exit.'); - const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH); - await pipeline.startChat(); - promptUser(); - - // Function to prompt the user for input - function promptUser() { - rl.question('> ', handleInput); - } - - // Function to handle user input - async function handleInput(input) { - input = input.trim(); - - // Check for exit command - if (input === 'finish') { - console.log('Goodbye!'); - await pipeline.finishChat(); - rl.close(); - process.exit(0); - } - - const result = await pipeline.generate(input, generationCallback); - console.log('\n'); - - // Wait for new input - promptUser(); - } - - function generationCallback(chunk) { - process.stdout.write(chunk); - } -} From ea0ff4a6aa8fb21db8f9b9b6b2bf5fa49a4ec73e Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 23 Dec 2024 12:12:16 +0100 Subject: [PATCH 46/47] Align linux workflow with js samples README --- .github/workflows/linux.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 44a659f132..750e2df69e 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -526,12 +526,8 @@ jobs: run: npm run test_setup working-directory: ${{ env.JS_SRC_DIR }} - - name: Link genai-node package - run: npm link - working-directory: ${{ env.JS_SRC_DIR }} - - - name: Install genai-node package to samples - run: npm link genai-node + - name: Install genai-node samples dependencies + run: npm install working-directory: ${{ env.JS_SAMPLES_DIR }} - name: Run tests From 7f8ee366755d2258a9b07f83f86846cca4f342fc Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 23 Dec 2024 16:42:21 +0100 Subject: [PATCH 47/47] Remove tbb specification from linux js workflow --- .github/workflows/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 750e2df69e..f54ff79738 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -458,7 +458,7 @@ jobs: - name: Build openvino + genai run: | cmake -DOPENVINO_EXTRA_MODULES=../openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ - -DCPACK_GENERATOR=NPM -DENABLE_JS=ON -UTBB* -DENABLE_SYSTEM_TBB=OFF \ + -DCPACK_GENERATOR=NPM \ -DENABLE_PYTHON=OFF \ -DENABLE_WHEEL=OFF \ -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \