diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 0bb0c1af6e..f54ff79738 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -42,7 +42,7 @@ jobs: runs-on: aks-linux-2-cores-8gb container: image: 'openvinogithubactions.azurecr.io/openvino_provider:0.1.0' - volumes: + volumes: - /mount:/mount - ${{ github.workspace }}:${{ github.workspace }} @@ -109,16 +109,16 @@ jobs: merge-multiple: true - name: CMake Build - run: | + run: | source ${{ env.OV_INSTALL_DIR }}/setupvars.sh cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ${{ env.SRC_DIR}} -B ${{ env.BUILD_DIR }} cmake --build ${{ env.BUILD_DIR}} --config ${{ matrix.build-type }} --parallel $(nproc) cmake --install ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --prefix ${{ env.INSTALL_DIR }} - + - name: Pack Artifacts run: tar -cvf - * | pigz > ${{ env.BUILD_DIR }}/${{ env.GENAI_ARCHIVE_NAME }} working-directory: ${{ env.INSTALL_DIR }} - + - name: Upload Archive Distribution Package if: ${{ always() }} uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 @@ -137,7 +137,7 @@ jobs: runs-on: aks-linux-4-cores-16gb container: image: openvinogithubactions.azurecr.io/ov_build/ubuntu_22_04_x64:${{ needs.openvino_download.outputs.docker_tag }} - volumes: + volumes: - /mount:/mount - ${{ github.workspace }}:${{ github.workspace }} options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING @@ -161,7 +161,7 @@ jobs: name: ${{ needs.openvino_download.outputs.ov_artifact_name }} path: ${{ env.OV_INSTALL_DIR }} merge-multiple: true - + - name: Build Tokenizers Wheel run: | python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} \ @@ -169,7 +169,7 @@ jobs: ${{ needs.openvino_download.outputs.ov_wheel_source }} \ ${{ env.SRC_DIR }}/thirdparty/openvino_tokenizers working-directory: ${{ env.OV_INSTALL_DIR }} - + - name: Build GenAI Wheel run: | python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} \ @@ -177,11 +177,11 @@ jobs: ${{ needs.openvino_download.outputs.ov_wheel_source }} \ ${{ env.SRC_DIR }} working-directory: ${{ env.OV_INSTALL_DIR }} - + - name: Build WWB Wheel run: python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} ${{ env.SRC_DIR }}/tools/who_what_benchmark working-directory: ${{ env.OV_INSTALL_DIR }} - + - name: Upload Wheels if: ${{ always() }} uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 @@ -189,7 +189,7 @@ jobs: name: genai_wheels path: ${{ env.INSTALL_DIR }} if-no-files-found: 'error' - + genai_build_samples: name: Build Samples - ${{ matrix.build-type }} strategy: @@ -204,7 +204,7 @@ jobs: runs-on: aks-linux-2-cores-8gb container: image: openvinogithubactions.azurecr.io/ov_build/ubuntu_22_04_x64:${{ needs.openvino_download.outputs.docker_tag }} - volumes: + volumes: - /mount:/mount - ${{ github.workspace }}:${{ github.workspace }} options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING @@ -228,17 +228,17 @@ jobs: pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_archive_${{ matrix.build-type }}}" path: ${{ env.OV_INSTALL_DIR }} merge-multiple: true - + - name: Extract Artifacts run: pigz -dc ${{ env.GENAI_ARCHIVE_NAME }} | tar -xf - -C ${{ env.OV_INSTALL_DIR }} working-directory: ${{ env.OV_INSTALL_DIR }} - + - name: Build Samples (Release) if: ${{ 'Release' == matrix.build-type }} run: | chmod +x ${{ env.OV_INSTALL_DIR }}/samples/cpp/build_samples.sh ${{ env.OV_INSTALL_DIR }}/samples/cpp/build_samples.sh -i ${{ env.INSTALL_DIR }} - + - name: Build Samples (${{ matrix.build-type }}) if: ${{ 'Release' != matrix.build-type }} run: | @@ -246,7 +246,7 @@ jobs: cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ${{ env.OV_INSTALL_DIR }}/samples/cpp/ -B ${{ env.BUILD_DIR }} cmake --build ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --parallel $(nproc) cmake --install ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --component samples_bin --prefix ${{ env.INSTALL_DIR }} - + - name: Pack Artifacts run: tar -cvf - * | pigz > ${{ env.INSTALL_DIR }}/${{ env.GENAI_SAMPLES_NAME }} working-directory: ${{ env.INSTALL_DIR }} @@ -258,7 +258,7 @@ jobs: name: genai_samples_${{ matrix.build-type }} path: ${{ env.INSTALL_DIR }}/*.tar.gz if-no-files-found: 'error' - + genai_tests_wheel: name: Python (${{ matrix.test.name}}) Tests (wheel) needs: [ openvino_download, genai_build_wheel ] @@ -277,7 +277,7 @@ jobs: runs-on: aks-linux-4-cores-16gb container: image: openvinogithubactions.azurecr.io/ov_test/ubuntu_22_04_x64:${{ needs.openvino_download.outputs.docker_tag }} - volumes: + volumes: - /mount:/mount - ${{ github.workspace }}:${{ github.workspace }} @@ -287,39 +287,39 @@ jobs: BUILD_DIR: ${{ github.workspace }}/build TRANSFORMERS_CACHE: ${{ github.workspace }}/models # Hugging Face transformers cache HF_HOME: ${{ github.workspace }}/datasets # Hugging Face datasets cache - + steps: - name: Clone openvino.genai uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: ${{ env.SRC_DIR }} submodules: recursive - + - name: Download Build Artifacts uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_wheels}" path: ${{ env.INSTALL_DIR }} merge-multiple: true - + - name: Install GenAI Wheels uses: ./src/.github/actions/install_wheel with: packages: "openvino;openvino_tokenizers[transformers];openvino_genai;whowhatbench" requirements_files: "${{ env.SRC_DIR }}/tests/python_tests/requirements.txt" local_wheel_dir: ${{ env.INSTALL_DIR }}/wheels - + - name: Tests run: python -m pytest -v ./${{ matrix.test.cmd }} working-directory: ${{ env.SRC_DIR }} - + genai_samples_tests: name: Samples Tests - ${{ matrix.build-type }} strategy: fail-fast: false matrix: build-type: [Release] - needs: [ openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples ] + needs: [ openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples ] timeout-minutes: 45 defaults: run: @@ -327,7 +327,7 @@ jobs: runs-on: aks-linux-2-cores-8gb container: image: openvinogithubactions.azurecr.io/ov_test/ubuntu_22_04_x64:${{ needs.openvino_download.outputs.docker_tag }} - volumes: + volumes: - /mount:/mount - ${{ github.workspace }}:${{ github.workspace }} @@ -336,41 +336,41 @@ jobs: SRC_DIR: ${{ github.workspace }}/src BUILD_DIR: ${{ github.workspace }}/build MODELS_DIR: ${{ github.workspace }}/models - + steps: - name: Clone openvino.genai uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: ${{ env.SRC_DIR }} submodules: recursive - + - name: Download Build Artifacts uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_archive_${{ matrix.build-type }},genai_samples_${{ matrix.build-type }},genai_wheels}" path: ${{ env.INSTALL_DIR }} merge-multiple: true - + - name: Extract Artifacts run: | pigz -dc ${{ env.GENAI_ARCHIVE_NAME }} | tar -xf - -C ${{ env.INSTALL_DIR }} pigz -dc ${{ env.GENAI_SAMPLES_NAME }} | tar -xf - -C ${{ env.INSTALL_DIR }} working-directory: ${{ env.INSTALL_DIR }} - + - name: Install Wheels uses: ./src/.github/actions/install_wheel with: packages: "openvino;openvino_tokenizers[transformers];openvino_genai" requirements_files: "${{ env.SRC_DIR }}/samples/requirements.txt" local_wheel_dir: ${{ env.INSTALL_DIR }}/wheels - + - name: Download & convert Models and data run: | mkdir -p ${{ env.MODELS_DIR }} optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 ${{ env.MODELS_DIR }}/TinyLlama-1.1B-Chat-v1.0 optimum-cli export openvino --trust-remote-code --model openai/whisper-tiny ${{ env.MODELS_DIR }}/whisper-tiny wget https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/librispeech_s5/how_are_you_doing_today.wav -O ${{ env.MODELS_DIR }}/how_are_you_doing_today.wav - + - name: Test multinomial_causal_lm.py if: ${{ 'Release' == matrix.build-type }} # Python bindings can be built in Release only timeout-minutes: 1 @@ -382,10 +382,10 @@ jobs: timeout-minutes: 1 run: ${{ env.INSTALL_DIR }}/samples/python/whisper_speech_recognition/whisper_speech_recognition.py ./whisper-tiny/ how_are_you_doing_today.wav working-directory: ${{ env.MODELS_DIR }} - + - name: C++ Tests Prerequisites run: python -m pip uninstall openvino openvino-tokenizers openvino-genai -y - + - name: Test greedy_causal_lm run: | source ${{ env.INSTALL_DIR }}/setupvars.sh @@ -398,9 +398,147 @@ jobs: ${{ env.INSTALL_DIR }}/samples_bin/whisper_speech_recognition ./whisper-tiny/ how_are_you_doing_today.wav working-directory: ${{ env.MODELS_DIR }} + genai_nodejs_bindings: + name: Produce genai nodejs binaries archive + timeout-minutes: 150 + defaults: + run: + shell: bash + runs-on: ubuntu-20.04-16-cores + env: + DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input + CMAKE_BUILD_TYPE: 'Release' + CMAKE_GENERATOR: 'Ninja Multi-Config' + CMAKE_CXX_COMPILER_LAUNCHER: ccache + CMAKE_C_COMPILER_LAUNCHER: ccache + OPENVINO_REPO: ${{ github.workspace }}/openvino + GENAI_REPO: ${{ github.workspace }}/openvino.genai + BUILD_DIR: ${{ github.workspace }}/build + CCACHE_DIR: ${{ github.workspace }}/ccache + CCACHE_MAXSIZE: 2000Mi + + steps: + - name: Set apt + run: | + echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null + echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null + echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null + echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null + + - name: Clone OpenVINO + uses: actions/checkout@v4 + with: + repository: 'openvinotoolkit/openvino' + path: ${{ env.OPENVINO_REPO }} + submodules: 'true' + ref: ${{ env.OV_BRANCH}} + + - name: Install build dependencies + run: | + sudo -E ${OPENVINO_REPO}/install_build_dependencies.sh + sudo apt-get install ccache + + - name: Clone GenAI + uses: actions/checkout@v4 + with: + path: ${{ env.GENAI_REPO }} + submodules: recursive + + - name: Setup ccache + uses: actions/cache@v4 + with: + # Should save cache only if run in the master branch of the base repo + # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push + save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} + path: ${{ env.CCACHE_DIR }} + key: ${{ runner.os }}-${{ runner.arch }}-ccache-ov-and-genai-${{ matrix.build-type }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-ccache-ov-and-genai-${{ matrix.build-type }} + + - name: Build openvino + genai + run: | + cmake -DOPENVINO_EXTRA_MODULES=../openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ + -DCPACK_GENERATOR=NPM \ + -DENABLE_PYTHON=OFF \ + -DENABLE_WHEEL=OFF \ + -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ + -S ./openvino -B ./build + cmake --build ./build --target package -j + + - name: Run javascript tests + working-directory: ${{ env.GENAI_REPO }}/src/js + run: | + mkdir bin + cp ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz bin + cd bin && tar -xzf genai_nodejs_bindings.tar.gz && rm genai_nodejs_bindings.tar.gz && cd .. + npm install + npm test + + # + # Upload build artifacts and logs + # + + - name: Upload genai nodejs bindings archive + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: genai_nodejs_bindings + path: ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz + if-no-files-found: 'error' + + genai_nodejs_samples_tests: + name: NodeJS Samples Tests + needs: [ genai_nodejs_bindings ] + timeout-minutes: 30 + defaults: + run: + shell: bash + runs-on: ubuntu-20.04-16-cores + env: + INSTALL_DIR: ${{ github.workspace }}/ov + GENAI_REPO: ${{ github.workspace }}/openvino.genai + MODELS_DIR: ${{ github.workspace }}/models + JS_SRC_DIR: ${{ github.workspace }}/openvino.genai/src/js + JS_SAMPLES_DIR: ${{ github.workspace }}/openvino.genai/samples/js/chat_sample + BIN_DIR: ${{ github.workspace }}/openvino.genai/src/js/bin + steps: + - name: Clone openvino.genai + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: ${{ env.GENAI_REPO }} + submodules: recursive + + - name: Download genai nodejs bindings archive + uses: actions/download-artifact@v4 + with: + name: genai_nodejs_bindings + path: ${{ env.BIN_DIR }} + + - name: Unpack genai nodejs bindings archive + run: tar -xzf genai_nodejs_bindings.tar.gz && rm genai_nodejs_bindings.tar.gz + working-directory: ${{ env.BIN_DIR }} + + - name: Install nodejs dependencies + run: npm install + working-directory: ${{ env.JS_SRC_DIR }} + + - name: Download model for tests + run: npm run test_setup + working-directory: ${{ env.JS_SRC_DIR }} + + - name: Install genai-node samples dependencies + run: npm install + working-directory: ${{ env.JS_SAMPLES_DIR }} + + - name: Run tests + run: npm test + env: + MODEL_PATH: ${{ env.JS_SRC_DIR }}/tests/models/Llama-3.2-3B-Instruct-openvino-8bit + working-directory: ${{ env.JS_SAMPLES_DIR }} + Overall_Status: name: ci/gha_overall_status_linux - needs: [openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples, genai_tests_wheel, genai_samples_tests] + needs: [openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples, genai_tests_wheel, genai_samples_tests, genai_nodejs_bindings] if: ${{ always() }} runs-on: ubuntu-latest steps: diff --git a/CMakeLists.txt b/CMakeLists.txt index fec8df34af..5cb2162cb5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -109,6 +109,9 @@ set(CPACK_COMPONENTS_ALL core_genai core_genai_dev cpp_samples_genai licensing_g if(ENABLE_PYTHON) list(APPEND CPACK_COMPONENTS_ALL pygenai_${Python3_VERSION_MAJOR}_${Python3_VERSION_MINOR}) endif() +if(ENABLE_JS) + list(APPEND CPACK_COMPONENTS_ALL genai_node_addon) +endif() if(WIN32 AND NOT DEFINED CPACK_GENERATOR) set(CPACK_GENERATOR "ZIP") endif() diff --git a/cmake/features.cmake b/cmake/features.cmake index 0434b21ee9..999ee1d81e 100644 --- a/cmake/features.cmake +++ b/cmake/features.cmake @@ -3,3 +3,4 @@ # option(ENABLE_PYTHON "Enable Python API build" ON) +option(ENABLE_JS "Enable JS API build" OFF) diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 02539df6e7..f26c97d61c 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -2,6 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 # +# Samples do not need to be built for NPM package +if(CPACK_GENERATOR STREQUAL "NPM") + return() +endif() + add_subdirectory(cpp/beam_search_causal_lm) add_subdirectory(cpp/benchmark_genai) add_subdirectory(cpp/chat_sample) diff --git a/samples/js/chat_sample/.gitignore b/samples/js/chat_sample/.gitignore new file mode 100644 index 0000000000..3c3629e647 --- /dev/null +++ b/samples/js/chat_sample/.gitignore @@ -0,0 +1 @@ +node_modules diff --git a/samples/js/chat_sample/README.md b/samples/js/chat_sample/README.md new file mode 100644 index 0000000000..46caba48e3 --- /dev/null +++ b/samples/js/chat_sample/README.md @@ -0,0 +1,48 @@ +# JavaScript chat_sample that supports most popular models like LLaMA 3 + +This example showcases inference of text-generation Large Language Models (LLMs): `chatglm`, `LLaMA`, `Qwen` and other models with the same signature. The application doesn't have many configuration options to encourage the reader to explore and modify the source code. For example, change the device for inference to GPU. The sample fearures `Pipeline.LLMPipeline` and configures it for the chat scenario. + +## Download and convert the model and tokenizers + +To convert model you have to use python package `optimum-intel`. +The `--upgrade-strategy eager` option is needed to ensure `optimum-intel` is upgraded to the latest version. + +Install [../../export-requirements.txt](../../export-requirements.txt) to convert a model. + +```sh +pip install --upgrade-strategy eager -r ../../export-requirements.txt +optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 +``` + +## Run: + +Compile GenAI JavaScript bindings archive first using the instructions in [../../../src/js/README.md](../../../src/js/README.md#build-bindings). + +Run `npm install` in current folder and then run the sample: + +`node chat_sample.js TinyLlama-1.1B-Chat-v1.0` + +Discrete GPUs (dGPUs) usually provide better performance compared to CPUs. It is recommended to run larger models on a dGPU with 32GB+ RAM. For example, the model meta-llama/Llama-2-13b-chat-hf can benefit from being run on a dGPU. Modify the source code to change the device for inference to the GPU. + +See https://github.com/openvinotoolkit/openvino.genai/blob/master/src/README.md#supported-models for the list of supported models. + +### Troubleshooting + +#### Unicode characters encoding error on Windows + +Example error: +``` +UnicodeEncodeError: 'charmap' codec can't encode character '\u25aa' in position 0: character maps to +``` + +If you encounter the error described in the example when sample is printing output to the Windows console, it is likely due to the default Windows encoding not supporting certain Unicode characters. To resolve this: +1. Enable Unicode characters for Windows cmd - open `Region` settings from `Control panel`. `Administrative`->`Change system locale`->`Beta: Use Unicode UTF-8 for worldwide language support`->`OK`. Reboot. +2. Enable UTF-8 mode by setting environment variable `PYTHONIOENCODING="utf8"`. + +#### Missing chat template + +If you encounter an exception indicating a missing "chat template" when launching the `ov::genai::LLMPipeline` in chat mode, it likely means the model was not tuned for chat functionality. To work this around, manually add the chat template to tokenizer_config.json of your model. +The following template can be used as a default, but it may not work properly with every model: +``` +"chat_template": "{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|im_start|>user\n' + message['content'] + '<|im_end|>\n<|im_start|>assistant\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|im_end|>\n'}}{% endif %}{% endfor %}", +``` diff --git a/samples/js/chat_sample/chat_sample.js b/samples/js/chat_sample/chat_sample.js new file mode 100644 index 0000000000..cf4c5e7704 --- /dev/null +++ b/samples/js/chat_sample/chat_sample.js @@ -0,0 +1,54 @@ +import readline from 'readline'; +import { Pipeline } from 'genai-node'; + +main(); + +function streamer(subword) { + process.stdout.write(subword); +} + +async function main() { + const MODEL_PATH = process.argv[2]; + + if (!MODEL_PATH) { + console.error('Please specify path to model directory\n' + + 'Run command must be: `node chat_sample.js *path_to_model_dir*`'); + process.exit(1); + } + + const device = 'CPU'; // GPU can be used as well + + // Create interface for reading user input from stdin + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + + const pipe = await Pipeline.LLMPipeline(MODEL_PATH, device); + const config = { 'max_new_tokens': 100 }; + + await pipe.startChat(); + promptUser(); + + // Function to prompt the user for input + function promptUser() { + rl.question('question:\n', handleInput); + } + + // Function to handle user input + async function handleInput(input) { + input = input.trim(); + + // Check for exit command + if (!input) { + await pipe.finishChat(); + rl.close(); + process.exit(0); + } + + await pipe.generate(input, config, streamer); + console.log('\n----------'); + + if (!rl.closed) promptUser(); + } +} diff --git a/samples/js/chat_sample/package-lock.json b/samples/js/chat_sample/package-lock.json new file mode 100644 index 0000000000..fbee0db012 --- /dev/null +++ b/samples/js/chat_sample/package-lock.json @@ -0,0 +1,42 @@ +{ + "name": "genai-node-demo", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "genai-node-demo", + "version": "1.0.0", + "license": "Apache-2.0", + "devDependencies": { + "genai-node": "../../../src/js/" + }, + "engines": { + "node": ">=21.0.0" + } + }, + "../../../src/js": { + "name": "genai-node", + "version": "2024.5.0-preview", + "dev": true, + "license": "Apache-2.0", + "os": [ + "linux", + "darwin", + "win32" + ], + "devDependencies": { + "@huggingface/hub": "^0.21.0", + "global-agent": "^3.0.0", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=21.0.0" + } + }, + "node_modules/genai-node": { + "resolved": "../../../src/js", + "link": true + } + } +} diff --git a/samples/js/chat_sample/package.json b/samples/js/chat_sample/package.json new file mode 100644 index 0000000000..24e66a120d --- /dev/null +++ b/samples/js/chat_sample/package.json @@ -0,0 +1,15 @@ +{ + "name": "genai-node-demo", + "version": "1.0.0", + "license": "Apache-2.0", + "type": "module", + "devDependencies": { + "genai-node": "../../../src/js/" + }, + "engines": { + "node": ">=21.0.0" + }, + "scripts": { + "test": "node tests/usage.test.js" + } +} diff --git a/samples/js/chat_sample/tests/usage.test.js b/samples/js/chat_sample/tests/usage.test.js new file mode 100644 index 0000000000..63856cbe43 --- /dev/null +++ b/samples/js/chat_sample/tests/usage.test.js @@ -0,0 +1,63 @@ +import { env } from 'process'; +import { spawn } from 'child_process'; + +const MODEL_PATH = env.MODEL_PATH; +const prompt = 'Tell me exactly, no changes, print as is: "Hello world"'; +const expected = 'Hello world'; + +if (!MODEL_PATH) + throw new Error( + 'Please environment variable MODEL_PATH to the path of the model directory' + ); + +const runTest = async () => { + return new Promise((resolve, reject) => { + const script = spawn('node', ['chat_sample.js', MODEL_PATH]); + let output = ''; + + // Collect output from stdout + script.stdout.on('data', (data) => { + output += data.toString(); + }); + + // Capture errors + script.stderr.on('data', (data) => { + reject(data.toString()); + }); + + // Send input after detecting the question prompt + script.stdout.once('data', (data) => { + if (data.toString().startsWith('question:')) { + script.stdin.write(`${prompt}\n`); // Provide input + script.stdin.end(); // Close stdin to signal EOF + } + }); + + // Check results when the process exits + script.on('close', (code) => { + if (code !== 0) { + return reject(`Process exited with code ${code}`); + } + + // Log the output + console.log(`Result output: ${output}`); + + // Validate the output + if (output.includes(expected)) { + resolve('Test passed!'); + } else { + reject('Test failed: Output did not match expected result.'); + } + }); + }); +}; + +runTest() + .then((message) => { + console.log(message); + process.exit(0); + }) + .catch((err) => { + console.error(err); + process.exit(1); + }); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d9f3cc64db..432d68cbef 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,3 +7,7 @@ add_subdirectory(cpp) if(ENABLE_PYTHON) add_subdirectory(python) endif() + +if(ENABLE_JS) + add_subdirectory(js) +endif() diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt index d02f32ded9..7864158e06 100644 --- a/src/cpp/CMakeLists.txt +++ b/src/cpp/CMakeLists.txt @@ -133,13 +133,42 @@ if(MSVC OR APPLE) set(ARCH_DIR ${ARCH_DIR}/${CMAKE_BUILD_TYPE}) endif() +# Put binaries at the top level for NPM package +if(CPACK_GENERATOR STREQUAL "NPM") + set(LIBRARY_DESTINATION .) + set(ARCHIVE_DESTINATION .) + set(RUNTIME_DESTINATION .) + + # setting RPATH / LC_RPATH depending on platform + if(LINUX) + # to find libopenvino.so in the same folder + set(rpaths "$ORIGIN") + elseif(APPLE) + # to find libopenvino.dylib in the same folder + set(rpaths "@loader_path") + endif() + + if(rpaths) + set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "${rpaths}") + endif() +else() + set(LIBRARY_DESTINATION runtime/lib/${ARCH_DIR}) + set(ARCHIVE_DESTINATION runtime/lib/${ARCH_DIR}) + set(RUNTIME_DESTINATION runtime/bin/${ARCH_DIR}) +endif() + install(TARGETS ${TARGET_NAME} EXPORT OpenVINOGenAITargets - LIBRARY DESTINATION runtime/lib/${ARCH_DIR} COMPONENT core_genai + LIBRARY DESTINATION ${LIBRARY_DESTINATION} COMPONENT core_genai NAMELINK_COMPONENT core_genai_dev - ARCHIVE DESTINATION runtime/lib/${ARCH_DIR} COMPONENT core_genai_dev - RUNTIME DESTINATION runtime/bin/${ARCH_DIR} COMPONENT core_genai + ARCHIVE DESTINATION ${ARCHIVE_DESTINATION} COMPONENT core_genai_dev + RUNTIME DESTINATION ${RUNTIME_DESTINATION} COMPONENT core_genai INCLUDES DESTINATION runtime/include) +# samples do not need to be built for NPM package +if(CPACK_GENERATOR STREQUAL "NPM") + return() +endif() + install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION runtime/include COMPONENT core_genai_dev) install(EXPORT OpenVINOGenAITargets FILE OpenVINOGenAITargets.cmake diff --git a/src/js/.gitignore b/src/js/.gitignore new file mode 100644 index 0000000000..638d8a5bb7 --- /dev/null +++ b/src/js/.gitignore @@ -0,0 +1,7 @@ +.vscode +bin +bin.* +build +thirdparty +node_modules +tests/models diff --git a/src/js/.npmignore b/src/js/.npmignore new file mode 100644 index 0000000000..9bf3e571b1 --- /dev/null +++ b/src/js/.npmignore @@ -0,0 +1,15 @@ +.vscode +bin.* +build +include +src +tests + +.eslintrc.js +CMakeLists.txt +tsconfig.json +TODO.md +build.sh + +**/*.tsbuildinfo +*.tgz diff --git a/src/js/CMakeLists.txt b/src/js/CMakeLists.txt new file mode 100644 index 0000000000..7e4ff0bea4 --- /dev/null +++ b/src/js/CMakeLists.txt @@ -0,0 +1,93 @@ +cmake_minimum_required(VERSION 3.18) + +# Set C++ standard +set(CMAKE_CXX_STANDARD 17) + +set(dist_folder "${CMAKE_SOURCE_DIR}/bin/") + +if(WIN32) + set(CMAKE_SHARED_LINKER_FLAGS /DELAYLOAD:NODE.EXE) + set(CMAKE_JS_LIB ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/node.lib) + set(CMAKE_JS_SRC ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/win_delay_load_hook.cc) + + set(CMAKE_JS_NODELIB_DEF ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/node-lib.def) + set(CMAKE_JS_NODELIB_TARGET ${CMAKE_JS_LIB}) + set(DELAYIMP_LIB delayimp.lib) +endif() + +project(genai_node_addon) + +# Specify NAPI version 8 +# supports v12.22.0+, v14.17.0+, v15.12.0+, 16.0.0 and all later Node.js versions +add_definitions(-DNAPI_VERSION=8) + +include(FetchContent) + +FetchContent_Declare( + node-api-headers + URL https://github.com/nodejs/node-api-headers/archive/refs/tags/v1.1.0.tar.gz + URL_HASH SHA256=70608bc1e6dddce280285f3462f18a106f687c0720a4b90893e1ecd86e5a8bbf +) +FetchContent_MakeAvailable(node-api-headers) + +FetchContent_Declare( + node-addon-api + URL https://github.com/nodejs/node-addon-api/archive/refs/tags/v8.0.0.tar.gz + URL_HASH SHA256=42424c5206b9d67b41af4fcff5d6e3cb22074168035a03b8467852938a281d47 +) +FetchContent_MakeAvailable(node-addon-api) + +# Create a library +add_library(${PROJECT_NAME} SHARED + ${CMAKE_CURRENT_SOURCE_DIR}/src/addon.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/llm_pipeline/llm_pipeline_wrapper.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/llm_pipeline/finish_chat_worker.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/llm_pipeline/start_chat_worker.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/llm_pipeline/init_worker.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/helper.cpp + + ${CMAKE_JS_SRC} +) + +# Include directories +target_include_directories(${PROJECT_NAME} PRIVATE + "${node-api-headers_SOURCE_DIR}/include" + "${node-addon-api_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}" +) + +target_link_libraries(${PROJECT_NAME} PRIVATE openvino::genai ${DELAYIMP_LIB} ${CMAKE_JS_LIB}) + +if(MSVC AND CMAKE_JS_NODELIB_DEF AND CMAKE_JS_NODELIB_TARGET) # Generate node.lib + execute_process(COMMAND ${CMAKE_AR} /def:${CMAKE_JS_NODELIB_DEF} /out:${CMAKE_JS_NODELIB_TARGET} ${CMAKE_STATIC_LINKER_FLAGS}) +endif() + +if(APPLE) + target_link_options(${PROJECT_NAME} PRIVATE -Wl,-undefined,suppress,-flat_namespace) +elseif(AARCH64 OR ARM) + target_link_options(${PROJECT_NAME} PRIVATE -Wl,--unresolved-symbols=ignore-all) +endif() + +# Set library properties +set_target_properties(${PROJECT_NAME} PROPERTIES + PREFIX "" + SUFFIX ".node" +) + +# setting RPATH / LC_RPATH depending on platform +if(LINUX) + # to find libopenvino_genai.so in the same folder + set(rpaths "$ORIGIN") +elseif(APPLE) + # to find libopenvino_genai.dylib in the same folder + set(rpaths "@loader_path") +endif() + +if(rpaths) + set_target_properties(${PROJECT_NAME} PROPERTIES INSTALL_RPATH "${rpaths}") +endif() + +install(TARGETS ${PROJECT_NAME} + LIBRARY DESTINATION . COMPONENT ${PROJECT_NAME} + RUNTIME DESTINATION . COMPONENT ${PROJECT_NAME} +) diff --git a/src/js/README.md b/src/js/README.md new file mode 100644 index 0000000000..7971c238fc --- /dev/null +++ b/src/js/README.md @@ -0,0 +1,56 @@ +we# OpenVINO™ GenAI Node.js bindings (preview) + +## DISCLAIMER + +This is preview version, do not use it in production! + +## Install and Run + +### Requirements + +- Node.js v21+ +- Tested on Ubuntu, another OS didn't tested yet + +### Build Bindings + +#### Build OpenVINO GenAI as OpenVINO Extra Module + +OpenVINO GenAI Node.js bindings can be built as an extra module during the OpenVINO build process. This method simplifies the build process by integrating OpenVINO GenAI directly into the OpenVINO build. + +1. Clone OpenVINO repository: + ```sh + git clone --recursive https://github.com/openvinotoolkit/openvino.git + ``` +1. Configure CMake with OpenVINO extra modules: + ```sh + cmake -DOPENVINO_EXTRA_MODULES=*absolute path to genai repository directory* -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ + -DCPACK_GENERATOR=NPM -DENABLE_JS=ON -UTBB* -DENABLE_SYSTEM_TBB=OFF \ + -DENABLE_PYTHON=OFF \ + -DENABLE_WHEEL=OFF \ + -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ + -S ./openvino -B ./build + ``` +1. Build OpenVINO archive with GenAI: + ```sh + cmake --build ./build --target package -j + ``` + +1. Put Node.js bindings into npm package `bin` directory and install dependencies: + ```sh + mkdir ./src/js/bin/ + tar -xvf ./build/genai_nodejs_bindings.tar.gz --directory ./src/js/bin/ + cd ./src/js/ + npm install + ``` +1. Run tests to be sure that everything works: + ```sh + npm test + ``` + +### Using as npm Dependency + +To use this package locally use `npm link` in `src/js/` directory +and `npm link genai-node` in the folder where you want to add this package as a dependency + +To extract this package and use it as distributed npm package run `npm package`. +This command creates archive that you may use in your projects. diff --git a/src/js/include/addon.hpp b/src/js/include/addon.hpp new file mode 100644 index 0000000000..35e5cc462e --- /dev/null +++ b/src/js/include/addon.hpp @@ -0,0 +1,20 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +typedef Napi::Function (*Prototype)(Napi::Env); + +struct AddonData { + Napi::FunctionReference core; +}; + +void init_class(Napi::Env env, + Napi::Object exports, + std::string class_name, + Prototype func, + Napi::FunctionReference& reference); + +Napi::Object init_module(Napi::Env env, Napi::Object exports); diff --git a/src/js/include/helper.hpp b/src/js/include/helper.hpp new file mode 100644 index 0000000000..4a010df019 --- /dev/null +++ b/src/js/include/helper.hpp @@ -0,0 +1,23 @@ +#pragma once +#include + +#include "openvino/core/type/element_type.hpp" +#include "openvino/openvino.hpp" + +ov::AnyMap to_anyMap(const Napi::Env&, const Napi::Value&); + +/** + * @brief Template function to convert Javascript data types into C++ data types + * @tparam TargetType destinated C++ data type + * @param info Napi::CallbackInfo contains all arguments passed to a function or method + * @param idx specifies index of a argument inside info. + * @return specified argument converted to a TargetType. + */ +template +TargetType js_to_cpp(const Napi::Env& env, const Napi::Value& value); + +/** @brief A template specialization for TargetType ov::Any */ +template <> +ov::Any js_to_cpp(const Napi::Env& env, const Napi::Value& value); + +bool is_napi_value_int(const Napi::Env& env, const Napi::Value& num); diff --git a/src/js/include/llm_pipeline/finish_chat_worker.hpp b/src/js/include/llm_pipeline/finish_chat_worker.hpp new file mode 100644 index 0000000000..ca80b30aff --- /dev/null +++ b/src/js/include/llm_pipeline/finish_chat_worker.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include "openvino/genai/llm_pipeline.hpp" + +using namespace Napi; + +class FinishChatWorker : public AsyncWorker { + public: + FinishChatWorker(Function& callback, std::shared_ptr& pipe); + virtual ~FinishChatWorker(){}; + + void Execute(); + void OnOK(); + + private: + std::shared_ptr& pipe; +}; diff --git a/src/js/include/llm_pipeline/init_worker.hpp b/src/js/include/llm_pipeline/init_worker.hpp new file mode 100644 index 0000000000..5fc05969fb --- /dev/null +++ b/src/js/include/llm_pipeline/init_worker.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include +#include "openvino/genai/llm_pipeline.hpp" + +using namespace Napi; + +class InitWorker : public AsyncWorker { + public: + InitWorker(Function& callback, std::shared_ptr& pipe, + const std::string model_path, std::string device); + virtual ~InitWorker(){}; + + void Execute(); + void OnOK(); + + private: + std::shared_ptr& pipe; + std::string model_path; + std::string device; +}; diff --git a/src/js/include/llm_pipeline/llm_pipeline_wrapper.hpp b/src/js/include/llm_pipeline/llm_pipeline_wrapper.hpp new file mode 100644 index 0000000000..872e9ea023 --- /dev/null +++ b/src/js/include/llm_pipeline/llm_pipeline_wrapper.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include "openvino/genai/llm_pipeline.hpp" + +class LLMPipelineWrapper : public Napi::ObjectWrap { +public: + LLMPipelineWrapper(const Napi::CallbackInfo& info); + + static Napi::Function get_class(Napi::Env env); + + Napi::Value init(const Napi::CallbackInfo& info); + Napi::Value generate(const Napi::CallbackInfo& info); + Napi::Value start_chat(const Napi::CallbackInfo& info); + Napi::Value finish_chat(const Napi::CallbackInfo& info); +private: + bool is_loaded = false; + bool is_initialized = false; + bool is_running = false; + + std::string model_path; + std::string device; + + std::shared_ptr pipe = nullptr; + std::function streamer; +}; diff --git a/src/js/include/llm_pipeline/start_chat_worker.hpp b/src/js/include/llm_pipeline/start_chat_worker.hpp new file mode 100644 index 0000000000..fde0cfaa0a --- /dev/null +++ b/src/js/include/llm_pipeline/start_chat_worker.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include "openvino/genai/llm_pipeline.hpp" + +using namespace Napi; + +class StartChatWorker : public AsyncWorker { + public: + StartChatWorker(Function& callback, std::shared_ptr& pipe); + virtual ~StartChatWorker(){}; + + void Execute(); + void OnOK(); + + private: + std::shared_ptr& pipe; +}; diff --git a/src/js/lib/bindings.cjs b/src/js/lib/bindings.cjs new file mode 100644 index 0000000000..acd9e590b8 --- /dev/null +++ b/src/js/lib/bindings.cjs @@ -0,0 +1 @@ +module.exports = require('../bin/genai_node_addon.node'); diff --git a/src/js/lib/module.js b/src/js/lib/module.js new file mode 100644 index 0000000000..b390712fa4 --- /dev/null +++ b/src/js/lib/module.js @@ -0,0 +1,118 @@ +import util from 'node:util'; + +import addon from './bindings.cjs'; + +class LLMPipeline { + modelPath = null; + device = null; + pipeline = null; + isInitialized = false; + isChatStarted = false; + + constructor(modelPath, device) { + this.modelPath = modelPath; + this.device = device; + } + + async init() { + if (this.isInitialized) + throw new Error('Pipeline is already initialized'); + + this.pipeline = new addon.LLMPipeline(); + + const init = util.promisify(this.pipeline.init.bind(this.pipeline)); + const result = await init(this.modelPath, this.device); + + this.isInitialized = true; + + return result; + } + + async startChat() { + if (this.isChatStarted) + throw new Error('Chat is already started'); + + const startChatPromise = util.promisify( + this.pipeline.startChat.bind(this.pipeline) + ); + const result = await startChatPromise(); + + this.isChatStarted = true; + + return result; + } + async finishChat() { + if (!this.isChatStarted) + throw new Error('Chat is not started'); + + const finishChatPromise = util.promisify( + this.pipeline.finishChat.bind(this.pipeline) + ); + const result = await finishChatPromise(); + + this.isChatStarted = false; + + return result; + } + + async generate(prompt, generationCallbackOrOptions, generationCallback) { + let options = {}; + + if (!generationCallback) + generationCallback = generationCallbackOrOptions; + else + options = generationCallbackOrOptions; + + if (!this.isInitialized) + throw new Error('Pipeline is not initialized'); + + if (typeof prompt !== 'string') + throw new Error('Prompt must be a string'); + if (typeof generationCallback !== 'function') + throw new Error('Generation callback must be a function'); + if (typeof options !== 'object') + throw new Error('Options must be an object'); + + let result = ''; + const castedOptions = {}; + + for (const key in options) castedOptions[key] = String(options[key]); + + const promise = new Promise((resolve, reject) => { + const generationCallbackDecorator = function(isDone, chunk) { + if (isDone) return resolve(result); + + result += chunk; + + try { + generationCallback(chunk); + } catch (err) { + reject(err); + } + }; + + try { + this.pipeline.generate(prompt, generationCallbackDecorator, castedOptions); + } catch (err) { + reject(err); + } + }); + + return promise; + } +} + +class Pipeline { + static async LLMPipeline(modelPath, device = 'CPU') { + const pipeline = new LLMPipeline(modelPath, device); + await pipeline.init(); + + return pipeline; + } +} + + +export { + addon, + Pipeline, +}; diff --git a/src/js/package-lock.json b/src/js/package-lock.json new file mode 100644 index 0000000000..4da5b57ea7 --- /dev/null +++ b/src/js/package-lock.json @@ -0,0 +1,470 @@ +{ + "name": "genai-node", + "version": "2024.5.0-preview", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "genai-node", + "version": "2024.5.0-preview", + "license": "Apache-2.0", + "os": [ + "linux", + "darwin", + "win32" + ], + "devDependencies": { + "@huggingface/hub": "^0.21.0", + "global-agent": "^3.0.0", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=21.0.0" + } + }, + "node_modules/@huggingface/hub": { + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/@huggingface/hub/-/hub-0.21.0.tgz", + "integrity": "sha512-DpitNhqobMJLTv8dUq/EMtrz1dpfs3UrSVCxe1aKpjLAdOs6Gm6rqrinUFNvC9G88RIRzIYzojUtYUqlkKwKnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@huggingface/tasks": "^0.13.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@huggingface/tasks": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.13.4.tgz", + "integrity": "sha512-LETHbMSK3gHBFU0D09ziEJm6t1Pcgii4SFwHw+d+8MFGfkAryxaDl2qaHY+PxiTkZEeaTLd6G8/239SJuVxyWg==", + "dev": true, + "license": "MIT" + }, + "node_modules/boolean": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", + "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "dev": true, + "license": "MIT" + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/define-properties": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", + "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", + "dev": true, + "license": "MIT", + "dependencies": { + "define-data-property": "^1.0.1", + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "dev": true, + "license": "MIT" + }, + "node_modules/es-define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", + "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-intrinsic": "^1.2.4" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", + "dev": true, + "license": "MIT" + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", + "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/global-agent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", + "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "es6-error": "^4.1.1", + "matcher": "^3.0.0", + "roarr": "^2.15.3", + "semver": "^7.3.2", + "serialize-error": "^7.0.1" + }, + "engines": { + "node": ">=10.0" + } + }, + "node_modules/globalthis": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", + "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "define-properties": "^1.2.1", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gopd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", + "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-intrinsic": "^1.1.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-proto": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", + "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", + "dev": true, + "license": "ISC" + }, + "node_modules/matcher": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", + "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", + "dev": true, + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "dev": true, + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/roarr": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", + "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "detect-node": "^2.0.4", + "globalthis": "^1.0.1", + "json-stringify-safe": "^5.0.1", + "semver-compare": "^1.0.0", + "sprintf-js": "^1.1.2" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/semver-compare": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", + "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", + "dev": true, + "license": "MIT" + }, + "node_modules/serialize-error": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", + "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", + "dev": true, + "license": "MIT", + "dependencies": { + "type-fest": "^0.13.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/type-fest": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + } + } +} diff --git a/src/js/package.json b/src/js/package.json new file mode 100644 index 0000000000..5b069fb01f --- /dev/null +++ b/src/js/package.json @@ -0,0 +1,30 @@ +{ + "name": "genai-node", + "type": "module", + "version": "2024.5.0-preview", + "description": "OpenVINO™ GenAI pipelines for using from Node.js environment", + "license": "Apache-2.0", + "main": "./lib/module.js", + "os": [ + "linux", + "darwin", + "win32" + ], + "engines": { + "node": ">=21.0.0" + }, + "keywords": [ + "OpenVINO", + "OpenVINO GenAI", + "GenAI" + ], + "scripts": { + "test_setup": "node ./tests/setup.js", + "test": "npm run test_setup && node --test ./tests/*.test.js" + }, + "devDependencies": { + "node-fetch": "^3.3.2", + "global-agent": "^3.0.0", + "@huggingface/hub": "^0.21.0" + } +} diff --git a/src/js/src/addon.cpp b/src/js/src/addon.cpp new file mode 100644 index 0000000000..4bd1da7bb6 --- /dev/null +++ b/src/js/src/addon.cpp @@ -0,0 +1,30 @@ +#include +#include + +#include "include/addon.hpp" + +#include "include/llm_pipeline/llm_pipeline_wrapper.hpp" + +void init_class(Napi::Env env, + Napi::Object exports, + std::string class_name, + Prototype func, + Napi::FunctionReference& reference) { + const auto& prototype = func(env); + + reference = Napi::Persistent(prototype); + exports.Set(class_name, prototype); +} + +// Define the addon initialization function +Napi::Object init_module(Napi::Env env, Napi::Object exports) { + auto addon_data = new AddonData(); + env.SetInstanceData(addon_data); + + init_class(env, exports, "LLMPipeline", &LLMPipelineWrapper::get_class, addon_data->core); + + return exports; +} + +// Register the addon with Node.js +NODE_API_MODULE(genai-node, init_module) diff --git a/src/js/src/helper.cpp b/src/js/src/helper.cpp new file mode 100644 index 0000000000..106994603b --- /dev/null +++ b/src/js/src/helper.cpp @@ -0,0 +1,53 @@ +#include "include/helper.hpp" + +ov::AnyMap to_anyMap(const Napi::Env& env, const Napi::Value& val) { + ov::AnyMap properties; + if (!val.IsObject()) { + OPENVINO_THROW("Passed Napi::Value must be an object."); + } + const auto& parameters = val.ToObject(); + const auto& keys = parameters.GetPropertyNames(); + + for (uint32_t i = 0; i < keys.Length(); ++i) { + const auto& property_name = static_cast(keys[i]).ToString().Utf8Value(); + + const auto& any_value = js_to_cpp(env, parameters.Get(property_name)); + + properties.insert(std::make_pair(property_name, any_value)); + } + + return properties; +} + +template <> +ov::Any js_to_cpp(const Napi::Env& env, const Napi::Value& value) { + if (value.IsString()) { + return ov::Any(value.ToString().Utf8Value()); + } else if (value.IsBigInt()) { + Napi::BigInt big_value = value.As(); + bool is_lossless; + int64_t big_num = big_value.Int64Value(&is_lossless); + + if (!is_lossless) { + OPENVINO_THROW("Result of BigInt conversion to int64_t results in a loss of precision"); + } + + return ov::Any(big_num); + } else if (value.IsNumber()) { + Napi::Number num = value.ToNumber(); + + if (is_napi_value_int(env, value)) { + return ov::Any(num.Int32Value()); + } else { + return ov::Any(num.DoubleValue()); + } + } else if (value.IsBoolean()) { + return ov::Any(value.ToBoolean()); + } else { + OPENVINO_THROW("Cannot convert to ov::Any"); + } +} + +bool is_napi_value_int(const Napi::Env& env, const Napi::Value& num) { + return env.Global().Get("Number").ToObject().Get("isInteger").As().Call({num}).ToBoolean().Value(); +} diff --git a/src/js/src/llm_pipeline/finish_chat_worker.cpp b/src/js/src/llm_pipeline/finish_chat_worker.cpp new file mode 100644 index 0000000000..b07284688c --- /dev/null +++ b/src/js/src/llm_pipeline/finish_chat_worker.cpp @@ -0,0 +1,14 @@ +#include "include/llm_pipeline/finish_chat_worker.hpp" +#include +#include + +FinishChatWorker::FinishChatWorker(Function& callback, std::shared_ptr& pipe) + : AsyncWorker(callback), pipe(pipe) {}; + +void FinishChatWorker::Execute() { + this->pipe->finish_chat(); +}; + +void FinishChatWorker::OnOK() { + Callback().Call({ Env().Null() }); +}; diff --git a/src/js/src/llm_pipeline/init_worker.cpp b/src/js/src/llm_pipeline/init_worker.cpp new file mode 100644 index 0000000000..87dd1aaf34 --- /dev/null +++ b/src/js/src/llm_pipeline/init_worker.cpp @@ -0,0 +1,18 @@ +#include "include/llm_pipeline/init_worker.hpp" +#include +#include + +InitWorker::InitWorker( + Function& callback, + std::shared_ptr& pipe, + const std::string model_path, + const std::string device +) : AsyncWorker(callback), pipe(pipe), model_path(model_path), device(device) {}; + +void InitWorker::Execute() { + this->pipe = std::make_shared(this->model_path, this->device); +}; + +void InitWorker::OnOK() { + Callback().Call({ Env().Null() }); +}; diff --git a/src/js/src/llm_pipeline/llm_pipeline_wrapper.cpp b/src/js/src/llm_pipeline/llm_pipeline_wrapper.cpp new file mode 100644 index 0000000000..df1cfceadc --- /dev/null +++ b/src/js/src/llm_pipeline/llm_pipeline_wrapper.cpp @@ -0,0 +1,151 @@ +#include "include/helper.hpp" + +#include "include/llm_pipeline/llm_pipeline_wrapper.hpp" +#include "include/llm_pipeline/start_chat_worker.hpp" +#include "include/llm_pipeline/finish_chat_worker.hpp" +#include "include/llm_pipeline/init_worker.hpp" + +struct TsfnContext { + TsfnContext(std::string prompt) : prompt(prompt) {}; + ~TsfnContext() { + // std::cout << "Tsfn destructed" << std::endl; + }; + + std::thread native_thread; + Napi::ThreadSafeFunction tsfn; + + std::string prompt; + std::shared_ptr pipe = nullptr; + std::shared_ptr options = nullptr; +}; + +void performInferenceThread(TsfnContext* context) { + auto callback = [](Napi::Env env, Napi::Function js_callback, TsfnContext* context) { + try { + std::function streamer = [env, js_callback](std::string word) { + js_callback.Call({ + Napi::Boolean::New(env, false), + Napi::String::New(env, word) + }); + + // Return flag corresponds whether generation should be stopped. + // false means continue generation. + return false; + }; + + ov::genai::GenerationConfig config; + + config.update_generation_config(*context->options); + + context->pipe->generate(context->prompt, config, streamer); + js_callback.Call({ + Napi::Boolean::New(env, true) + }); + } catch(std::exception& err) { + Napi::Error::Fatal("performInferenceThread callback error. Details:" , err.what()); + } + }; + + try { + napi_status status = context->tsfn.BlockingCall(context, callback); + if (status != napi_ok) { + // Handle error + Napi::Error::Fatal("performInferenceThread error", "napi_status != napi_ok"); + } + + context->tsfn.Release(); + } + catch(std::exception& e) { + Napi::Error::Fatal("performInferenceThread error" , e.what()); + + context->tsfn.Release(); + } +} + +LLMPipelineWrapper::LLMPipelineWrapper(const Napi::CallbackInfo& info) : Napi::ObjectWrap(info) {}; + +Napi::Function LLMPipelineWrapper::get_class(Napi::Env env) { + return DefineClass(env, + "LLMPipeline", + {InstanceMethod("init", &LLMPipelineWrapper::init), + InstanceMethod("generate", &LLMPipelineWrapper::generate), + InstanceMethod("startChat", &LLMPipelineWrapper::start_chat), + InstanceMethod("finishChat", &LLMPipelineWrapper::finish_chat)}); +} + +Napi::Value LLMPipelineWrapper::init(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + const std::string model_path = info[0].ToString(); + const std::string device = info[1].ToString(); + Napi::Function callback = info[2].As(); + + InitWorker* asyncWorker = new InitWorker(callback, this->pipe, model_path, device); + asyncWorker->Queue(); + + return info.Env().Undefined(); +} + +Napi::Value LLMPipelineWrapper::generate(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + TsfnContext* context = nullptr; + + try { + std::string prompt = info[0].ToString(); + ov::AnyMap options; + if (info.Length() == 3) { + options = to_anyMap(info.Env(), info[2]); + } + + context = new TsfnContext(prompt); + context->pipe = this->pipe; + context->options = std::make_shared(options); + // Create a ThreadSafeFunction + context->tsfn = Napi::ThreadSafeFunction::New( + env, + info[1].As(), // JavaScript function called asynchronously + "TSFN", // Name + 0, // Unlimited queue + 1, // Only one thread will use this initially + [context](Napi::Env) { // Finalizer used to clean threads up + // std::cout << "Finalize TFSN" << std::endl; + context->native_thread.join(); + delete context; + } + ); + context->native_thread = std::thread(performInferenceThread, context); + + return Napi::Boolean::New(env, false); + } catch(Napi::TypeError& type_err) { + throw type_err; + } catch(std::exception& err) { + std::cout << "Catch in the thread: '" << err.what() << "'" << std::endl; + if (context != nullptr) { + context->tsfn.Release(); + } + + throw Napi::Error::New(env, err.what()); + } + + return Napi::Boolean::New(env, true); +} + +Napi::Value LLMPipelineWrapper::start_chat(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + Napi::Function callback = info[0].As(); + + StartChatWorker* asyncWorker = new StartChatWorker(callback, this->pipe); + asyncWorker->Queue(); + + return info.Env().Undefined(); +} + +Napi::Value LLMPipelineWrapper::finish_chat(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + + Napi::Function callback = info[0].As(); + + FinishChatWorker* asyncWorker = new FinishChatWorker(callback, this->pipe); + asyncWorker->Queue(); + + return info.Env().Undefined(); +} diff --git a/src/js/src/llm_pipeline/start_chat_worker.cpp b/src/js/src/llm_pipeline/start_chat_worker.cpp new file mode 100644 index 0000000000..302c505105 --- /dev/null +++ b/src/js/src/llm_pipeline/start_chat_worker.cpp @@ -0,0 +1,14 @@ +#include "include/llm_pipeline/start_chat_worker.hpp" +#include +#include + +StartChatWorker::StartChatWorker(Function& callback, std::shared_ptr& pipe) + : AsyncWorker(callback), pipe(pipe) {}; + +void StartChatWorker::Execute() { + this->pipe->start_chat(); +}; + +void StartChatWorker::OnOK() { + Callback().Call({ Env().Null() }); +}; diff --git a/src/js/tests/bindings.test.js b/src/js/tests/bindings.test.js new file mode 100644 index 0000000000..f76c45e0d9 --- /dev/null +++ b/src/js/tests/bindings.test.js @@ -0,0 +1,73 @@ +import addon from '../lib/bindings.cjs'; + +import assert from 'node:assert'; +import { describe, it, before, after } from 'node:test'; +import { models } from './models.js'; + +const MODEL_PATH = process.env.MODEL_PATH + || `./tests/models/${models[0].split('/')[1]}`; + +describe('bindings', () => { + let pipeline = null; + + before((_, done) => { + pipeline = new addon.LLMPipeline(); + + pipeline.init(MODEL_PATH, 'CPU', (err) => { + if (err) { + console.error(err); + process.exit(1); + } + + pipeline.startChat((err) => { + if (err) { + console.error(err); + process.exit(1); + } + + done(); + }); + }); + }); + + after((_, done) => { + pipeline.finishChat((err) => { + if (err) { + console.error(err); + process.exit(1); + } + + done(); + }); + }); + + it('should generate string result', (_, done) => { + let output = ''; + + pipeline.generate('Say Hello', (isDone, chunk) => { + if (!isDone) { + output += chunk; + + return; + } + }, { temperature: '0', max_new_tokens: '4' }); + + assert.ok(true); + done(); + }); + + it('should generate "Hello world"', (_, done) => { + let output = ''; + + pipeline.generate('Type "Hello world!" in English', (isDone, chunk) => { + if (!isDone) { + output += chunk; + + return; + } + + assert.strictEqual(output, 'Hello world!'); + done(); + }, { temperature: '0', max_new_tokens: '4' }); + }); +}); diff --git a/src/js/tests/models.js b/src/js/tests/models.js new file mode 100644 index 0000000000..b7f7505464 --- /dev/null +++ b/src/js/tests/models.js @@ -0,0 +1,3 @@ +export const models = [ + 'AIFunOver/Llama-3.2-3B-Instruct-openvino-8bit', +]; diff --git a/src/js/tests/module.test.js b/src/js/tests/module.test.js new file mode 100644 index 0000000000..cbb9e905a4 --- /dev/null +++ b/src/js/tests/module.test.js @@ -0,0 +1,142 @@ +import { Pipeline } from '../lib/module.js'; + +import assert from 'node:assert/strict'; +import { describe, it, before, after } from 'node:test'; +import { models } from './models.js'; + +const MODEL_PATH = process.env.MODEL_PATH + || `./tests/models/${models[0].split('/')[1]}`; + +describe('module', async () => { + let pipeline = null; + + await before(async () => { + pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'CPU'); + + await pipeline.startChat(); + }); + + await after(async () => { + await pipeline.finishChat(); + }); + + await it('should generate "Hello world"', async () => { + const result = await pipeline.generate( + 'Type "Hello world!" in English', + { temperature: '0', max_new_tokens: '4' }, + () => {}, + ); + + assert.strictEqual(result, 'Hello world!'); + }); +}); + +describe('corner cases', async () => { + it('should throw an error if pipeline is already initialized', async () => { + const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'CPU'); + + await assert.rejects( + async () => await pipeline.init(), + { + name: 'Error', + message: 'Pipeline is already initialized', + }, + ); + }); + + it('should throw an error if chat is already started', async () => { + const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'CPU'); + + await pipeline.startChat(); + + await assert.rejects( + () => pipeline.startChat(), + { + name: 'Error', + message: 'Chat is already started', + }, + ); + }); + + it('should throw an error if chat is not started', async () => { + const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'CPU'); + + await assert.rejects( + () => pipeline.finishChat(), + { + name: 'Error', + message: 'Chat is not started', + }, + ); + }); +}); + +describe('generation parameters validation', () => { + let pipeline = null; + + before(async () => { + pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'CPU'); + + await pipeline.startChat(); + }); + + after(async () => { + await pipeline.finishChat(); + }); + + it('should throw an error if temperature is not a number', async () => { + await assert.rejects( + async () => await pipeline.generate(), + { + name: 'Error', + message: 'Prompt must be a string', + }, + ); + }); + + it('should throw an error if generationCallback is not a function', async () => { + const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'CPU'); + + await pipeline.startChat(); + + await assert.rejects( + async () => await pipeline.generate('prompt'), + { + name: 'Error', + message: 'Generation callback must be a function', + }, + ); + }); + + it('should throw an error if options specified but not an object', async () => { + await assert.rejects( + async () => await pipeline.generate('prompt', 'options', () => {}), + { + name: 'Error', + message: 'Options must be an object', + }, + ); + }); + + it('should perform generation with default options', async () => { + try { + await pipeline.generate('prompt', { max_new_tokens: 1 }, () => {}); + } catch (error) { + assert.fail(error); + } + + assert.ok(true); + }); + + it('should return a string as generation result', async () => { + const reply = await pipeline.generate('prompt', { max_new_tokens: 1 }, () => {}); + + assert.strictEqual(typeof reply, 'string'); + }); + + it('should call generationCallback with string chunk', async () => { + await pipeline.generate('prompt', { max_new_tokens: 1 }, (chunk) => { + assert.strictEqual(typeof chunk, 'string'); + }); + }); +}); diff --git a/src/js/tests/setup.js b/src/js/tests/setup.js new file mode 100644 index 0000000000..3b52651719 --- /dev/null +++ b/src/js/tests/setup.js @@ -0,0 +1,6 @@ +import { dowloadModel } from './utils.js'; +import { models } from './models.js'; + +for (const model of models) { + await dowloadModel(model); +} diff --git a/src/js/tests/utils.js b/src/js/tests/utils.js new file mode 100644 index 0000000000..504782d8d1 --- /dev/null +++ b/src/js/tests/utils.js @@ -0,0 +1,47 @@ +import { bootstrap } from 'global-agent'; +import { promises as fs } from 'node:fs'; +import { listFiles, downloadFile } from '@huggingface/hub'; + +const BASE_DIR = './tests/models/'; + +bootstrap(); + +export async function dowloadModel(repo) { + console.log(`Downloading model '${repo}'`); + + const fetch = await import('node-fetch'); + const modelName = repo.split('/')[1]; + const destDir = `${BASE_DIR}${modelName}`; + + await fs.mkdir(destDir, { recursive: true }); + + const fileList = await listFiles({ + repo, + fetch: fetch.default, + }); + const fileNames = []; + for await (const file of fileList) { + fileNames.push(file.path); + } + + for (const path of fileNames) { + console.log(`Downloading file '${path}'`); + const response = await downloadFile({ + repo, + path, + fetch: fetch.default, + }); + const filename = `${destDir}/${path}`; + + await saveFile(filename, response); + console.log(`File '${path}' downloaded`); + } + + console.log(`Model '${repo}' downloaded`); +} + +async function saveFile(file, response) { + const arrayBuffer = await response.arrayBuffer(); + + await fs.writeFile(file, Buffer.from(arrayBuffer)); +}