From 0662eeb4f30066df70de95b0fcf3b010de77c342 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Tue, 16 Jul 2024 10:44:22 -0700 Subject: [PATCH] Make custom patch to get applied only once after sub module is checked out Signed-off-by: Heemin Kim --- .github/workflows/CI.yml | 2 -- .github/workflows/test_security.yml | 2 -- CHANGELOG.md | 1 + jni/cmake/init-faiss.cmake | 8 +------- jni/cmake/init-nmslib.cmake | 8 +------- scripts/windowsScript.ps1 | 7 +++++++ 6 files changed, 10 insertions(+), 18 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 0b9b24d983..e6dec4daff 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -37,8 +37,6 @@ jobs: steps: - name: Checkout k-NN uses: actions/checkout@v1 - with: - submodules: true # Setup git user so that patches for native libraries can be applied and committed - name: Setup git user diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index e0f2dbf451..77b726a69d 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -37,8 +37,6 @@ jobs: steps: - name: Checkout k-NN uses: actions/checkout@v1 - with: - submodules: true # Setup git user so that patches for native libraries can be applied and committed - name: Setup git user run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index e03e06ccbe..2f244e2a25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Release memory properly for an array type [#1820](https://github.com/opensearch-project/k-NN/pull/1820) * FIX Same Suffix Cause Recall Drop to zero [#1802](https://github.com/opensearch-project/k-NN/pull/1802) ### Infrastructure +* Apply custom patch only once after submodule is checked out [#1833](https://github.com/opensearch-project/k-NN/pull/1833) ### Documentation * Update dev guide to fix clang linking issue on arm [#1746](https://github.com/opensearch-project/k-NN/pull/1746) ### Maintenance diff --git a/jni/cmake/init-faiss.cmake b/jni/cmake/init-faiss.cmake index befed47037..ecc7de9f46 100644 --- a/jni/cmake/init-faiss.cmake +++ b/jni/cmake/init-faiss.cmake @@ -10,13 +10,7 @@ find_path(FAISS_REPO_DIR NAMES faiss PATHS ${CMAKE_CURRENT_SOURCE_DIR}/external/ if (NOT EXISTS ${FAISS_REPO_DIR}) message(STATUS "Could not find faiss. Pulling updated submodule.") execute_process(COMMAND git submodule update --init -- external/faiss WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) -endif () - -# Check if patch exist, this is to skip git apply during CI build. See CI.yml with ubuntu. -find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch 0002-Enable-precomp-table-to-be-shared-ivfpq.patch 0003-Custom-patch-to-support-range-search-params.patch 0004-Custom-patch-to-support-binary-vector.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss NO_DEFAULT_PATH) -# If it exists, apply patches -if (EXISTS ${PATCH_FILE}) message(STATUS "Applying custom patches.") execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) @@ -25,7 +19,7 @@ if (EXISTS ${PATCH_FILE}) if(RESULT_CODE) message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") endif() -endif() +endif () if (${CMAKE_SYSTEM_NAME} STREQUAL Darwin) if(CMAKE_C_COMPILER_ID MATCHES "Clang\$") diff --git a/jni/cmake/init-nmslib.cmake b/jni/cmake/init-nmslib.cmake index 387dce6bc6..9c03b5d6a6 100644 --- a/jni/cmake/init-nmslib.cmake +++ b/jni/cmake/init-nmslib.cmake @@ -10,13 +10,7 @@ find_path(NMS_REPO_DIR NAMES similarity_search PATHS ${CMAKE_CURRENT_SOURCE_DIR} if (NOT EXISTS ${NMS_REPO_DIR}) message(STATUS "Could not find nmslib. Pulling updated submodule.") execute_process(COMMAND git submodule update --init -- external/nmslib WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) -endif () - -# Check if patch exist, this is to skip git apply during CI build. See CI.yml with ubuntu. -find_path(PATCH_FILE NAMES 0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch 0002-Adds-ability-to-pass-ef-parameter-in-the-query-for-h.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib NO_DEFAULT_PATH) -# If it exists, apply patches -if (EXISTS ${PATCH_FILE}) message(STATUS "Applying custom patches.") execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0002-Adds-ability-to-pass-ef-parameter-in-the-query-for-h.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) @@ -24,6 +18,6 @@ if (EXISTS ${PATCH_FILE}) if(RESULT_CODE) message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") endif() -endif() +endif () add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib/similarity_search) diff --git a/scripts/windowsScript.ps1 b/scripts/windowsScript.ps1 index 2b0d4f7991..bfefa90b74 100644 --- a/scripts/windowsScript.ps1 +++ b/scripts/windowsScript.ps1 @@ -4,7 +4,14 @@ # git submodule update --init -- jni/external/nmslib +git -C jni/external/nmslib apply --3way --ignore-space-change --ignore-whitespace ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch +git -C jni/external/nmslib apply --3way --ignore-space-change --ignore-whitespace ../../patches/nmslib/0002-Adds-ability-to-pass-ef-parameter-in-the-query-for-h.patch + git submodule update --init -- jni/external/faiss +git -C jni/external/faiss apply --3way --ignore-space-change --ignore-whitespace ../../patches/nmslib/0001-Custom-patch-to-support-multi-vector.patch +git -C jni/external/faiss apply --3way --ignore-space-change --ignore-whitespace ../../patches/nmslib/0002-Enable-precomp-table-to-be-shared-ivfpq.patch +git -C jni/external/faiss apply --3way --ignore-space-change --ignore-whitespace ../../patches/nmslib/0003-Custom-patch-to-support-range-search-params.patch +git -C jni/external/faiss apply --3way --ignore-space-change --ignore-whitespace ../../patches/nmslib/0004-Custom-patch-to-support-binary-vector.patch # _MSC_VER is a predefined macro which defines the version of Visual Studio Compiler # As we are using x86_64-w64-mingw32-gcc compiler we need to replace this macro with __MINGW32__