diff --git a/.github/workflows/benchmark-linux.yaml b/.github/workflows/benchmark-linux.yaml index fe2a696e..a65209b5 100644 --- a/.github/workflows/benchmark-linux.yaml +++ b/.github/workflows/benchmark-linux.yaml @@ -21,7 +21,7 @@ jobs: - name: Build id: build - run: sudo su -c "PG_VERSION=15 USE_SOURCE=1 ./ci/scripts/build-linux.sh" + run: sudo su -c "PG_VERSION=15 USE_SOURCE=1 ./ci/scripts/build.sh" env: BRANCH_NAME: ${{ github.head_ref || github.ref_name }} diff --git a/.github/workflows/build-linux.yaml b/.github/workflows/build.yaml similarity index 68% rename from .github/workflows/build-linux.yaml rename to .github/workflows/build.yaml index b76fbd9c..36276a12 100644 --- a/.github/workflows/build-linux.yaml +++ b/.github/workflows/build.yaml @@ -16,51 +16,40 @@ on: required: false default: false jobs: - ubuntu-build: + build: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - include: - - postgres: 16 - os: ubuntu-22.04 - - postgres: 15 - os: ubuntu-22.04 - - postgres: 14 - os: ubuntu-22.04 - - postgres: 13 - os: ubuntu-22.04 - - postgres: 12 - os: ubuntu-22.04 - - postgres: 11 - os: ubuntu-22.04 + os: [ubuntu-22.04, macos-13] + postgres: [11, 12, 13, 14, 15, 16] steps: - uses: actions/checkout@v3 with: submodules: "recursive" - name: Build id: build - run: sudo su -c "PG_VERSION=$PG_VERSION USE_SOURCE=1 GITHUB_OUTPUT=$GITHUB_OUTPUT BUILD_PACKAGES=1 ./ci/scripts/build-linux.sh" + run: sudo sh -c "PG_VERSION=$PG_VERSION USE_SOURCE=1 GITHUB_OUTPUT=$GITHUB_OUTPUT BUILD_PACKAGES=1 ./ci/scripts/build.sh" env: PG_VERSION: ${{ matrix.postgres }} - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - name: Create Archive Package id: archive - run: sudo su -c "GITHUB_OUTPUT=$GITHUB_OUTPUT ./ci/scripts/package-archive.sh" - - name: Upload artifacts + run: sudo sh -c "GITHUB_OUTPUT=$GITHUB_OUTPUT ./ci/scripts/package-archive.sh" + - name: Upload deb package artifacts uses: actions/upload-artifact@v3 + if: ${{ steps.build.outputs.deb_package_path != '' }} with: - name: ${{ steps.build.outputs.package_name }} - path: ${{ steps.build.outputs.package_path }} + name: ${{ steps.build.outputs.deb_package_name }} + path: ${{ steps.build.outputs.deb_package_path }} - name: Upload archive package artifacts uses: actions/upload-artifact@v3 - if: ${{ steps.archive.outputs.archive_package_name != '' }} + if: ${{ steps.archive.outputs.archive_package_path != '' }} with: name: lantern-package path: ${{ steps.archive.outputs.archive_package_path }} - ubuntu-package: + package: runs-on: ubuntu-22.04 - needs: [ubuntu-build] + needs: [build] steps: - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 @@ -69,7 +58,7 @@ jobs: path: /tmp/lantern-package - name: Create universal package id: package - run: sudo su -c "GITHUB_OUTPUT=$GITHUB_OUTPUT PACKAGE_EXTRAS=1 GITHUB_TOKEN=$GITHUB_TOKEN ./ci/scripts/universal-package.sh" + run: sudo sh -c "GITHUB_OUTPUT=$GITHUB_OUTPUT PACKAGE_EXTRAS=1 GITHUB_TOKEN=$GITHUB_TOKEN ./ci/scripts/universal-package.sh" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Upload artifacts diff --git a/.github/workflows/test-linux.yaml b/.github/workflows/test.yaml similarity index 71% rename from .github/workflows/test-linux.yaml rename to .github/workflows/test.yaml index b5259633..11547b42 100644 --- a/.github/workflows/test-linux.yaml +++ b/.github/workflows/test.yaml @@ -16,31 +16,20 @@ on: required: false default: false jobs: - ubuntu-build: + test: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - include: - - postgres: 16 - os: ubuntu-22.04 - - postgres: 15 - os: ubuntu-22.04 - - postgres: 14 - os: ubuntu-22.04 - - postgres: 13 - os: ubuntu-22.04 - - postgres: 12 - os: ubuntu-22.04 - - postgres: 11 - os: ubuntu-22.04 + os: [ubuntu-22.04, macos-13] + postgres: [11, 12, 13, 14, 15, 16] steps: - uses: actions/checkout@v3 with: submodules: "recursive" - name: Build id: build - run: sudo su -c "PG_VERSION=$PG_VERSION USE_SOURCE=1 GITHUB_OUTPUT=$GITHUB_OUTPUT ENABLE_COVERAGE=1 ./ci/scripts/build-linux.sh" + run: sudo sh -c "PG_VERSION=$PG_VERSION USE_SOURCE=1 GITHUB_OUTPUT=$GITHUB_OUTPUT ENABLE_COVERAGE=1 ./ci/scripts/build.sh" env: PG_VERSION: ${{ matrix.postgres }} BRANCH_NAME: ${{ github.head_ref || github.ref_name }} @@ -48,11 +37,18 @@ jobs: - name: Setup tmate session uses: mxschmitt/action-tmate@v3 if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - - name: Run tests - id: test - run: sudo su postgres -c "PG_VERSION=$PG_VERSION ./ci/scripts/run-tests.sh" + - name: Run tests linux + id: test-linux + run: sudo su postgres -c "PG_VERSION=$PG_VERSION ./ci/scripts/run-tests-linux.sh" env: PG_VERSION: ${{ matrix.postgres }} + if: ${{ startsWith(matrix.os, 'ubuntu') }} + - name: Run tests mac + id: test-mac + run: ./ci/scripts/run-tests-mac.sh + env: + PG_VERSION: ${{ matrix.postgres }} + if: ${{ startsWith(matrix.os, 'mac') }} - name: Upload Postgres logs if: failure() uses: actions/upload-artifact@v3 @@ -65,7 +61,7 @@ jobs: uses: codecov/codecov-action@v3 env: codecov_token: ${{ secrets.CODECOV_TOKEN }} - if: ${{ env.codecov_token != '' && matrix.postgres == 15 }} # for now run only on once + if: ${{ env.codecov_token != '' && startsWith(matrix.os, 'ubuntu') && matrix.postgres == 15 }} # for now run only on once with: token: ${{ secrets.CODECOV_TOKEN }} name: codecov-lanterndb diff --git a/ci/scripts/build-docker.sh b/ci/scripts/build-docker.sh index 38ebc73c..bf087aac 100755 --- a/ci/scripts/build-docker.sh +++ b/ci/scripts/build-docker.sh @@ -1,11 +1,7 @@ #!/bin/bash get_cmake_flags(){ - # TODO:: remove after test echo "-DUSEARCH_NO_MARCH_NATIVE=ON" - # if [[ $ARCH == *"arm"* ]]; then - # echo "-DUSEARCH_NO_MARCH_NATIVE=ON" - # fi } export DEBIAN_FRONTEND=noninteractive diff --git a/ci/scripts/build-linux.sh b/ci/scripts/build-linux.sh index 341c27d4..334b6dfc 100755 --- a/ci/scripts/build-linux.sh +++ b/ci/scripts/build-linux.sh @@ -1,14 +1,6 @@ #!/bin/bash set -e -function setup_environment() { - export BRANCH=${BRANCH_NAME:-dev} - export POSTGRES_USER=postgres - export DEBIAN_FRONTEND=noninteractive - export PG_VERSION=${PG_VERSION:-15} - export GITHUB_OUTPUT=${GITHUB_OUTPUT:-/dev/null} -} - function setup_locale_and_install_packages() { echo "LC_ALL=en_US.UTF-8" > /etc/environment echo "en_US.UTF-8 UTF-8" > /etc/locale.gen @@ -22,60 +14,15 @@ function setup_locale_and_install_packages() { function setup_postgres() { # Add postgresql apt repo - export ARCH=$(dpkg-architecture -q DEB_BUILD_ARCH) echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - # Install postgres and dev files for C headers apt update apt install -y postgresql-$PG_VERSION postgresql-server-dev-$PG_VERSION - # Install pgvector - pushd /tmp - PGVECTOR_VERSION=0.5.0 - wget -O pgvector.tar.gz https://github.com/pgvector/pgvector/archive/refs/tags/v${PGVECTOR_VERSION}.tar.gz - tar xzf pgvector.tar.gz - pushd pgvector-${PGVECTOR_VERSION} - make && make install - popd - popd # Fix pg_config (sometimes it points to wrong version) rm -f /usr/bin/pg_config && ln -s /usr/lib/postgresql/$PG_VERSION/bin/pg_config /usr/bin/pg_config } -function clone_or_use_source() { - if [ -z ${USE_SOURCE} ]; then - # Clone from git - cd /tmp - git clone --recursive https://github.com/lanterndata/lantern.git -b $BRANCH - else - # Use already checkouted code - shopt -s dotglob - mkdir -p /tmp/lantern - cp -r ./* /tmp/lantern/ - fi -} - -function build_and_install() { - cd /tmp/lantern - mkdir build - cd build - - # TODO:: remove after test - flags="-DUSEARCH_NO_MARCH_NATIVE=ON" - # Treat warnings as errors in CI/CD - flags+=" -DCMAKE_COMPILE_WARNING_AS_ERROR=ON" - # if [[ $ARCH == *"arm"* ]]; then - # echo "-DUSEARCH_NO_MARCH_NATIVE=ON" - # fi - if [ -n "$ENABLE_COVERAGE" ] - then - flags="$flags -DCMAKE_C_COMPILER=/usr/bin/gcc -DCODECOVERAGE=ON" - fi - - # Run cmake - cmake $flags .. - make install -} - function package_if_necessary() { if [ -n "$BUILD_PACKAGES" ]; then # Bundle debian packages @@ -84,18 +31,13 @@ function package_if_necessary() { export EXT_VERSION=$(cmake --system-information | awk -F= '$1~/CMAKE_PROJECT_VERSION:STATIC/{print$2}') && \ export PACKAGE_NAME=lantern-${EXT_VERSION}-postgres-${PG_VERSION}-${ARCH}.deb && \ - echo "package_version=$EXT_VERSION" >> "$GITHUB_OUTPUT" && \ - echo "package_name=$PACKAGE_NAME" >> "$GITHUB_OUTPUT" && \ - echo "package_path=$(pwd)/$(ls *.deb | tr -d '\n')" >> "$GITHUB_OUTPUT" + echo "deb_package_name=$PACKAGE_NAME" >> "$GITHUB_OUTPUT" && \ + echo "deb_package_path=$(pwd)/$(ls *.deb | tr -d '\n')" >> "$GITHUB_OUTPUT" fi } -setup_environment -setup_locale_and_install_packages -setup_postgres -clone_or_use_source -build_and_install -package_if_necessary +function cleanup_environment() { + # Chown to postgres for running tests + chown -R postgres:postgres /tmp/lantern +} -# Chown to postgres for running tests -chown -R postgres:postgres /tmp/lantern diff --git a/ci/scripts/build-mac.sh b/ci/scripts/build-mac.sh new file mode 100755 index 00000000..ce39eb77 --- /dev/null +++ b/ci/scripts/build-mac.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -e + +function setup_locale_and_install_packages() { + export PATH="/usr/local/opt/postgresql@${PG_VERSION}/bin:$PATH" + export C_INCLUDE_PATH=/usr/local/include + export CPLUS_INCLUDE_PATH=/usr/local/include +} + +function setup_postgres() { + cmd="brew install postgresql@${PG_VERSION} clang-format" + if [[ $USER == "root" ]] + then + # Runner is github CI user + sh -c "sudo -u runner -i $cmd" + else + sh -c $cmd + fi +} + +function package_if_necessary() { + : + # TODO make and publish homebrew formula +} + +function cleanup_environment() { + # Add permission to for running tests from runner user + chmod -R 777 /tmp/lantern +} + diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh new file mode 100755 index 00000000..5330d915 --- /dev/null +++ b/ci/scripts/build.sh @@ -0,0 +1,80 @@ +#!/bin/bash +set -e + +# This sets $ARCH and $PLATFORM env variables +source "$(dirname "$0")/../../scripts/get_arch_and_platform.sh" + +if [[ $PLATFORM == "mac" ]]; then + BUILD_SCRIPT="build-mac.sh" +elif [[ $PLATFORM == "linux" ]]; then + BUILD_SCRIPT="build-linux.sh" +else + echo "Invalid target use one of [mac, linux]" + exit 1 +fi + +function setup_environment() { + export BRANCH=${BRANCH_NAME:-dev} + export POSTGRES_USER=postgres + export DEBIAN_FRONTEND=noninteractive + export PG_VERSION=${PG_VERSION:-15} + export GITHUB_OUTPUT=${GITHUB_OUTPUT:-/dev/null} +} + +function clone_or_use_source() { + if [ -z ${USE_SOURCE} ]; then + # Clone from git + cd /tmp + git clone --recursive https://github.com/lanterndata/lantern.git -b $BRANCH + else + # Use already checkouted code + shopt -s dotglob + rm -rf /tmp/lantern + mkdir -p /tmp/lantern + cp -r -P ./* /tmp/lantern/ + fi +} + +function install_external_dependencies() { + # Install pgvector + pushd /tmp + PGVECTOR_VERSION=0.5.0 + wget -O pgvector.tar.gz https://github.com/pgvector/pgvector/archive/refs/tags/v${PGVECTOR_VERSION}.tar.gz + tar xzf pgvector.tar.gz + pushd pgvector-${PGVECTOR_VERSION} + make && make install + popd + popd +} + +function build_and_install() { + cd /tmp/lantern + mkdir build + cd build + + flags="-DUSEARCH_NO_MARCH_NATIVE=ON" + + # Treat warnings as errors in CI/CD + flags+=" -DCMAKE_COMPILE_WARNING_AS_ERROR=ON" + + if [ -n "$ENABLE_COVERAGE" ] + then + flags="$flags -DCMAKE_C_COMPILER=/usr/bin/gcc -DCODECOVERAGE=ON" + fi + + # Run cmake + cmake $flags .. + make install +} + +# Source platform specific build script +source "$(dirname "$0")/${BUILD_SCRIPT}" + +setup_environment +setup_locale_and_install_packages +setup_postgres +install_external_dependencies +clone_or_use_source +build_and_install +package_if_necessary +cleanup_environment diff --git a/ci/scripts/package-archive.sh b/ci/scripts/package-archive.sh index 4b74552b..151f8dee 100755 --- a/ci/scripts/package-archive.sh +++ b/ci/scripts/package-archive.sh @@ -1,9 +1,6 @@ #!/bin/bash -if [ -z "$GITHUB_OUTPUT" ] -then - export GITHUB_OUTPUT=/dev/null -fi +GITHUB_OUTPUT=${GITHUB_OUTPUT:-/dev/null} cd /tmp/lantern/build && make archive cat /tmp/gh-output.txt >> $GITHUB_OUTPUT && rm -rf /tmp/gh-output diff --git a/ci/scripts/run-benchmarks.sh b/ci/scripts/run-benchmarks.sh index 0e342cea..2b52e5b3 100755 --- a/ci/scripts/run-benchmarks.sh +++ b/ci/scripts/run-benchmarks.sh @@ -18,9 +18,9 @@ wait_for_pg(){ export WORKDIR=/tmp/lantern export PG_VERSION=15 export GITHUB_OUTPUT=/dev/null -export PGDATA=/etc/postgresql/$PG_VERSION/main/ +export PGDATA=/etc/postgresql/$PG_VERSION/main -echo "port = 5432" >> $PGDATA/postgresql.conf +echo "port = 5432" >> ${PGDATA}/postgresql.conf # Enable auth without password echo "local all all trust" > $PGDATA/pg_hba.conf echo "host all all 127.0.0.1/32 trust" >> $PGDATA/pg_hba.conf diff --git a/ci/scripts/run-tests.sh b/ci/scripts/run-tests-linux.sh similarity index 72% rename from ci/scripts/run-tests.sh rename to ci/scripts/run-tests-linux.sh index c2553a1e..3ede685a 100755 --- a/ci/scripts/run-tests.sh +++ b/ci/scripts/run-tests-linux.sh @@ -1,6 +1,12 @@ #!/bin/bash set -e +WORKDIR=/tmp/lantern +GITHUB_OUTPUT=${GITHUB_OUTPUT:-/dev/null} +PG_VERSION=${PG_VERSION:-15} + +export PGDATA=/etc/postgresql/$PG_VERSION/main + wait_for_pg(){ tries=0 until pg_isready -U postgres 2>/dev/null; do @@ -15,21 +21,8 @@ wait_for_pg(){ done } -export WORKDIR=/tmp/lantern - -if [ -z "$PG_VERSION" ] -then - export PG_VERSION=15 -fi - -if [ -z "$GITHUB_OUTPUT" ] -then - export GITHUB_OUTPUT=/dev/null -fi - -export PGDATA=/etc/postgresql/$PG_VERSION/main/ # Set port -echo "port = 5432" >> $PGDATA/postgresql.conf +echo "port = 5432" >> ${PGDATA}/postgresql.conf # Run postgres database GCOV_PREFIX=$WORKDIR/build/CMakeFiles/lantern.dir/ GCOV_PREFIX_STRIP=5 POSTGRES_HOST_AUTH_METHOD=trust /usr/lib/postgresql/$PG_VERSION/bin/postgres 1>/tmp/pg-out.log 2>/tmp/pg-error.log & # Wait for start and run tests diff --git a/ci/scripts/run-tests-mac.sh b/ci/scripts/run-tests-mac.sh new file mode 100755 index 00000000..eef9af45 --- /dev/null +++ b/ci/scripts/run-tests-mac.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -e + +WORKDIR=/tmp/lantern +PG_VERSION=${PG_VERSION:-15} +export PATH="/usr/local/opt/postgresql@${PG_VERSION}/bin:$PATH" + +wait_for_pg(){ + tries=0 + until pg_isready -U postgres 2>/dev/null; do + if [ $tries -eq 10 ]; + then + echo "Can not connect to postgres" + exit 1 + fi + + sleep 1 + tries=$((tries+1)) + done +} + +# Start database +brew services start postgresql@$PG_VERSION + +wait_for_pg && cd $WORKDIR/build && make test diff --git a/ci/scripts/universal-package.sh b/ci/scripts/universal-package.sh index 0f098855..80bf2eea 100755 --- a/ci/scripts/universal-package.sh +++ b/ci/scripts/universal-package.sh @@ -1,10 +1,6 @@ #!/bin/bash -if [ -z "$GITHUB_OUTPUT" ] -then - export GITHUB_OUTPUT=/dev/null -fi - +GITHUB_OUTPUT=${GITHUB_OUTPUT:-/dev/null} PACKAGE_DIR=/tmp/lantern-package PACKAGE_VERSION=$(ls -t $PACKAGE_DIR | head -1 | sed -E "s#lantern-(.*)-postgres.*#\1#") PACKAGE_NAME=lantern-${PACKAGE_VERSION} @@ -15,9 +11,15 @@ mkdir $OUTPUT_DIR cd $PACKAGE_DIR for f in $(find "." -name "*.tar"); do current_archive_name=$(echo $f | sed -E 's#(.*).tar#\1#') - current_pg_version=$(echo $current_archive_name | sed -E 's#(.*)-postgres-(.*)-(.*)#\2#') - current_arch=$(echo $current_archive_name | sed -E 's#(.*)-postgres-(.*)-(.*)#\3#') - current_dest_folder=${OUTPUT_DIR}/src/${current_arch}/${current_pg_version} + current_pg_version=$(echo $current_archive_name | sed -E 's#(.*)-postgres-(.*)-(.*)-(.*)#\2#') + current_platform=$(echo $current_archive_name | sed -E 's#(.*)-postgres-(.*)-(.*)-(.*)#\3#') + current_arch=$(echo $current_archive_name | sed -E 's#(.*)-postgres-(.*)-(.*)-(.*)#\4#') + current_dest_folder=${OUTPUT_DIR}/src/${current_arch}/${current_platform}/${current_pg_version} + echo "current_archive_name=${current_archive_name}" + echo "current_pg_version=${current_pg_version}" + echo "current_arch=${current_arch}" + echo "current_platform=${current_platform}" + echo "current_dest_folder=${current_dest_folder}" tar xf $f diff --git a/scripts/get_arch_and_platform.sh b/scripts/get_arch_and_platform.sh new file mode 100755 index 00000000..3de1e773 --- /dev/null +++ b/scripts/get_arch_and_platform.sh @@ -0,0 +1,24 @@ +if command -v uname &> /dev/null +then + ARCH=$(uname -m) +elif command -v dpkg &> /dev/null +then + ARCH=$(dpkg --print-architecture) +elif command -v apk &> /dev/null +then + ARCH=$(apk --print-arch) +else + echo "Could not detect system architecture. Please specify with ARCH env variable" + exit 1 +fi + +# At this moment we only support mac and linux +unameOut="$(uname -s)" +case "${unameOut}" in + Linux*) PLATFORM=linux;; + Darwin*) PLATFORM=mac;; + *) PLATFORM=${unameOut} +esac + +export ARCH +export PLATFORM diff --git a/scripts/package.sh b/scripts/package.sh index e99bb414..607d534f 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -1,9 +1,9 @@ #!/bin/bash +source "$(dirname "$0")/get_arch_and_platform.sh" cd $BUILD_DIR -ARCH=$(dpkg-architecture -q DEB_BUILD_ARCH) EXT_VERSION=$(cmake --system-information | awk -F= '$1~/CMAKE_PROJECT_VERSION:STATIC/{print$2}') -PACKAGE_NAME=lantern-${EXT_VERSION}-postgres-${PG_VERSION}-${ARCH} +PACKAGE_NAME=lantern-${EXT_VERSION}-postgres-${PG_VERSION}-${PLATFORM}-${ARCH} mkdir -p ${BUILD_DIR}/${PACKAGE_NAME}/src cp ${SOURCE_DIR}/scripts/packaging/* ${BUILD_DIR}/${PACKAGE_NAME}/ diff --git a/scripts/packaging/install.sh b/scripts/packaging/install.sh index e5ecf4dc..45821827 100755 --- a/scripts/packaging/install.sh +++ b/scripts/packaging/install.sh @@ -10,23 +10,27 @@ then PG_CONFIG=$(which pg_config) fi -if [ -z "$ARCH" ] +if command -v uname &> /dev/null then - if command -v dpkg &> /dev/null - then - ARCH=$(dpkg --print-architecture) - elif command -v apk &> /dev/null - then - ARCH=$(apk --print-arch) - elif command -v uname &> /dev/null - then - ARCH=$(uname -m) - else - echo "Could not detect system architecture. Please specify with ARCH env variable" - exit - fi + ARCH=$(uname -m) +elif command -v dpkg &> /dev/null +then + ARCH=$(dpkg --print-architecture) +elif command -v apk &> /dev/null +then + ARCH=$(apk --print-arch) +else + echo "Could not detect system architecture. Please specify with ARCH env variable" + exit 1 fi +unameOut="$(uname -s)" +case "${unameOut}" in + Linux*) PLATFORM=linux;; + Darwin*) PLATFORM=mac;; + *) PLATFORM=${unameOut} +esac + PG_LIBRARY_DIR=$($PG_CONFIG --pkglibdir) PG_EXTENSION_DIR=$($PG_CONFIG --sharedir)/extension PG_VERSION_STRING=$($PG_CONFIG --version) @@ -38,13 +42,19 @@ then exit fi -if [ ! -d src/${ARCH}/${PG_VERSION} ] +if [ ! -d src/${ARCH}/${PLATFORM} ] +then + echo "Platform $PLATFORM not supported. Try building from source" + exit +fi + +if [ ! -d src/${ARCH}/${PLATFORM}/${PG_VERSION} ] then echo "Postgres version $PG_VERSION not supported" exit fi -cp -r src/${ARCH}/${PG_VERSION}/*.so $PG_LIBRARY_DIR +cp -r src/${ARCH}/${PLATFORM}/${PG_VERSION}/*.so $PG_LIBRARY_DIR cp -r shared/*.sql $PG_EXTENSION_DIR cp -r shared/*.control $PG_EXTENSION_DIR diff --git a/scripts/run_all_tests.sh b/scripts/run_all_tests.sh index 595b5cdd..de9df735 100755 --- a/scripts/run_all_tests.sh +++ b/scripts/run_all_tests.sh @@ -65,7 +65,7 @@ then fi # Check if pgvector is available -pgvector_installed=$($PSQL -U $DB_USER -d postgres -c "SELECT 1 FROM pg_available_extensions WHERE name = 'vector'" -tA) +pgvector_installed=$($PSQL -U $DB_USER -d postgres -c "SELECT 1 FROM pg_available_extensions WHERE name = 'vector'" -tA | tail -n 1 | tr -d '\n') # Generate schedule.txt rm -rf $TMP_OUTDIR/schedule.txt diff --git a/src/hnsw.c b/src/hnsw.c index 080d8116..7dad7bd4 100644 --- a/src/hnsw.c +++ b/src/hnsw.c @@ -211,7 +211,7 @@ static bool hnswvalidate(Oid opclassoid) * See https://www.postgresql.org/docs/current/index-api.html */ PGDLLEXPORT PG_FUNCTION_INFO_V1(hnsw_handler); -Datum hnsw_handler(PG_FUNCTION_ARGS) +Datum hnsw_handler(PG_FUNCTION_ARGS __attribute__((unused))) { IndexAmRoutine *amroutine = makeNode(IndexAmRoutine); @@ -395,4 +395,4 @@ float4 *DatumGetSizedFloatArray(Datum datum, HnswColumnType type, int dimensions } else { elog(ERROR, "Unsupported type"); } -} \ No newline at end of file +} diff --git a/test/expected/hnsw_vector.out b/test/expected/hnsw_vector.out index e4706f9b..ba2d8ea1 100644 --- a/test/expected/hnsw_vector.out +++ b/test/expected/hnsw_vector.out @@ -76,20 +76,20 @@ SELECT * FROM ldb_get_indexes('small_world'); INSERT INTO small_world (v) VALUES ('[99,99,2]'); INSERT INTO small_world (v) VALUES (NULL); -- Distance functions -SELECT id, ROUND(l2sq_dist(v, '[0,1,0]'::VECTOR)::numeric, 2) as dist +SELECT ROUND(l2sq_dist(v, '[0,1,0]'::VECTOR)::numeric, 2) as dist FROM small_world ORDER BY v <-> '[0,1,0]'::VECTOR LIMIT 7; - id | dist ------+------ - 010 | 0.00 - 011 | 1.00 - 000 | 1.00 - 110 | 1.00 - 001 | 2.00 - 111 | 2.00 - 100 | 2.00 + dist +------ + 0.00 + 1.00 + 1.00 + 1.00 + 2.00 + 2.00 + 2.00 (7 rows) -EXPLAIN (COSTS FALSE) SELECT id, ROUND(l2sq_dist(v, '[0,1,0]'::VECTOR)::numeric, 2) as dist +EXPLAIN (COSTS FALSE) SELECT ROUND(l2sq_dist(v, '[0,1,0]'::VECTOR)::numeric, 2) as dist FROM small_world ORDER BY v <-> '[0,1,0]'::VECTOR LIMIT 7; QUERY PLAN --------------------------------------------------------- @@ -98,20 +98,20 @@ FROM small_world ORDER BY v <-> '[0,1,0]'::VECTOR LIMIT 7; Order By: (v <-> '[0,1,0]'::vector) (3 rows) -SELECT id, ROUND(vector_l2sq_dist(v, '[0,1,0]'::VECTOR)::numeric, 2) as dist +SELECT ROUND(vector_l2sq_dist(v, '[0,1,0]'::VECTOR)::numeric, 2) as dist FROM small_world ORDER BY v <-> '[0,1,0]'::VECTOR LIMIT 7; - id | dist ------+------ - 010 | 0.00 - 011 | 1.00 - 000 | 1.00 - 110 | 1.00 - 001 | 2.00 - 111 | 2.00 - 100 | 2.00 + dist +------ + 0.00 + 1.00 + 1.00 + 1.00 + 2.00 + 2.00 + 2.00 (7 rows) -EXPLAIN (COSTS FALSE) SELECT id, ROUND(vector_l2sq_dist(v, '[0,1,0]'::VECTOR)::numeric, 2) as dist +EXPLAIN (COSTS FALSE) SELECT ROUND(vector_l2sq_dist(v, '[0,1,0]'::VECTOR)::numeric, 2) as dist FROM small_world ORDER BY v <-> '[0,1,0]'::VECTOR LIMIT 7; QUERY PLAN --------------------------------------------------------- @@ -127,7 +127,6 @@ CREATE INDEX ON large_vector USING lantern_hnsw (v); ERROR: vector dimension 2001 is too large. LanternDB currently supports up to 2000dim vectors \set ON_ERROR_STOP on -- Validate that index creation works with a larger number of vectors -SET client_min_messages=debug5; CREATE TABLE sift_base10k ( id SERIAL PRIMARY KEY, v VECTOR(128) @@ -139,13 +138,6 @@ INFO: inserted 10000 elements INFO: done saving 10000 vectors SELECT v AS v4444 FROM sift_base10k WHERE id = 4444 \gset EXPLAIN (COSTS FALSE) SELECT * FROM sift_base10k ORDER BY v <-> :'v4444' LIMIT 10; -DEBUG: LANTERN - Query cost estimator -DEBUG: LANTERN - --------------------- -DEBUG: LANTERN - Total cost: 1192.827449 -DEBUG: LANTERN - Selectivity: 1.000000 -DEBUG: LANTERN - Num pages: 292.000000 -DEBUG: LANTERN - Num tuples: 3333.000000 -DEBUG: LANTERN - --------------------- QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Limit @@ -158,13 +150,6 @@ SET hnsw.init_k = 4; WITH neighbors AS ( SELECT * FROM small_world order by v <-> '[1,0,0]' LIMIT 3 ) SELECT COUNT(*) from neighbors; -DEBUG: LANTERN - Query cost estimator -DEBUG: LANTERN - --------------------- -DEBUG: LANTERN - Total cost: 4.015000 -DEBUG: LANTERN - Selectivity: 1.000000 -DEBUG: LANTERN - Num pages: 1.000000 -DEBUG: LANTERN - Num tuples: 2.000000 -DEBUG: LANTERN - --------------------- count ------- 3 @@ -173,13 +158,6 @@ DEBUG: LANTERN - --------------------- WITH neighbors AS ( SELECT * FROM small_world order by v <-> '[1,0,0]' LIMIT 15 ) SELECT COUNT(*) from neighbors; -DEBUG: LANTERN - Query cost estimator -DEBUG: LANTERN - --------------------- -DEBUG: LANTERN - Total cost: 4.015000 -DEBUG: LANTERN - Selectivity: 1.000000 -DEBUG: LANTERN - Num pages: 1.000000 -DEBUG: LANTERN - Num tuples: 2.000000 -DEBUG: LANTERN - --------------------- count ------- 9 diff --git a/test/sql/hnsw_vector.sql b/test/sql/hnsw_vector.sql index ce62a43a..10a3e3b6 100644 --- a/test/sql/hnsw_vector.sql +++ b/test/sql/hnsw_vector.sql @@ -34,14 +34,14 @@ INSERT INTO small_world (v) VALUES ('[99,99,2]'); INSERT INTO small_world (v) VALUES (NULL); -- Distance functions -SELECT id, ROUND(l2sq_dist(v, '[0,1,0]'::VECTOR)::numeric, 2) as dist +SELECT ROUND(l2sq_dist(v, '[0,1,0]'::VECTOR)::numeric, 2) as dist FROM small_world ORDER BY v <-> '[0,1,0]'::VECTOR LIMIT 7; -EXPLAIN (COSTS FALSE) SELECT id, ROUND(l2sq_dist(v, '[0,1,0]'::VECTOR)::numeric, 2) as dist +EXPLAIN (COSTS FALSE) SELECT ROUND(l2sq_dist(v, '[0,1,0]'::VECTOR)::numeric, 2) as dist FROM small_world ORDER BY v <-> '[0,1,0]'::VECTOR LIMIT 7; -SELECT id, ROUND(vector_l2sq_dist(v, '[0,1,0]'::VECTOR)::numeric, 2) as dist +SELECT ROUND(vector_l2sq_dist(v, '[0,1,0]'::VECTOR)::numeric, 2) as dist FROM small_world ORDER BY v <-> '[0,1,0]'::VECTOR LIMIT 7; -EXPLAIN (COSTS FALSE) SELECT id, ROUND(vector_l2sq_dist(v, '[0,1,0]'::VECTOR)::numeric, 2) as dist +EXPLAIN (COSTS FALSE) SELECT ROUND(vector_l2sq_dist(v, '[0,1,0]'::VECTOR)::numeric, 2) as dist FROM small_world ORDER BY v <-> '[0,1,0]'::VECTOR LIMIT 7; -- Verify that index creation on a large vector produces an error @@ -51,8 +51,6 @@ CREATE INDEX ON large_vector USING lantern_hnsw (v); \set ON_ERROR_STOP on -- Validate that index creation works with a larger number of vectors -SET client_min_messages=debug5; - CREATE TABLE sift_base10k ( id SERIAL PRIMARY KEY, v VECTOR(128) @@ -79,4 +77,4 @@ SELECT ARRAY[1,2,3] <-> ARRAY[3,2,1]; -- Expect error due to mismatching vector dimensions SELECT 1 FROM small_world ORDER BY v <-> '[0,1,0,1]' LIMIT 1; -SELECT vector_l2sq_dist('[1,1]'::vector, '[0,1,0]'::vector); \ No newline at end of file +SELECT vector_l2sq_dist('[1,1]'::vector, '[0,1,0]'::vector); diff --git a/third_party/usearch b/third_party/usearch index ccc326e8..9c80e20c 160000 --- a/third_party/usearch +++ b/third_party/usearch @@ -1 +1 @@ -Subproject commit ccc326e8d96f548b124661077998698cc743c2fc +Subproject commit 9c80e20c1397beed1186d31e31a7962fc6109f81