Add Mac builds into CI pipeline

.github/workflows/benchmark-linux.yaml

@@ -21,7 +21,7 @@ jobs:
 
       - name: Build
         id: build
-        run: sudo su -c "PG_VERSION=15 USE_SOURCE=1 ./ci/scripts/build-linux.sh"
+        run: sudo su -c "PG_VERSION=15 USE_SOURCE=1 ./ci/scripts/build.sh"
         env:
           BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
 

.github/workflows/build-linux.yaml → .github/workflows/build.yaml

@@ -16,51 +16,40 @@ on:
         required: false
         default: false
 jobs:
-  ubuntu-build:
+  build:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - postgres: 16
-            os: ubuntu-22.04
-          - postgres: 15
-            os: ubuntu-22.04
-          - postgres: 14
-            os: ubuntu-22.04
-          - postgres: 13
-            os: ubuntu-22.04
-          - postgres: 12
-            os: ubuntu-22.04
-          - postgres: 11
-            os: ubuntu-22.04
+        os: [ubuntu-22.04, macos-13]
+        postgres: [11, 12, 13, 14, 15, 16]
     steps:
       - uses: actions/checkout@v3
         with:
           submodules: "recursive"
       - name: Build
         id: build
-        run: sudo su -c "PG_VERSION=$PG_VERSION USE_SOURCE=1 GITHUB_OUTPUT=$GITHUB_OUTPUT BUILD_PACKAGES=1 ./ci/scripts/build-linux.sh"
+        run: sudo sh -c "PG_VERSION=$PG_VERSION USE_SOURCE=1 GITHUB_OUTPUT=$GITHUB_OUTPUT BUILD_PACKAGES=1 ./ci/scripts/build.sh"
         env:
           PG_VERSION: ${{ matrix.postgres }}
-          BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
       - name: Create Archive Package
         id: archive
-        run: sudo su -c "GITHUB_OUTPUT=$GITHUB_OUTPUT ./ci/scripts/package-archive.sh"
-      - name: Upload artifacts
+        run: sudo sh -c "GITHUB_OUTPUT=$GITHUB_OUTPUT ./ci/scripts/package-archive.sh"
+      - name: Upload deb package artifacts
         uses: actions/upload-artifact@v3
+        if: ${{ steps.build.outputs.deb_package_path != '' }}
         with:
-          name: ${{ steps.build.outputs.package_name }}
-          path: ${{ steps.build.outputs.package_path }}
+          name: ${{ steps.build.outputs.deb_package_name }}
+          path: ${{ steps.build.outputs.deb_package_path }}
       - name: Upload archive package artifacts
         uses: actions/upload-artifact@v3
-        if: ${{ steps.archive.outputs.archive_package_name != '' }}
+        if: ${{ steps.archive.outputs.archive_package_path != '' }}
         with:
           name: lantern-package
           path: ${{ steps.archive.outputs.archive_package_path }}
-  ubuntu-package:
+  package:
     runs-on: ubuntu-22.04
-    needs: [ubuntu-build]
+    needs: [build]
     steps:
       - uses: actions/checkout@v3
       - uses: actions/download-artifact@v3
@@ -69,7 +58,7 @@ jobs:
           path: /tmp/lantern-package
       - name: Create universal package
         id: package
-        run: sudo su -c "GITHUB_OUTPUT=$GITHUB_OUTPUT PACKAGE_EXTRAS=1 GITHUB_TOKEN=$GITHUB_TOKEN ./ci/scripts/universal-package.sh"
+        run: sudo sh -c "GITHUB_OUTPUT=$GITHUB_OUTPUT PACKAGE_EXTRAS=1 GITHUB_TOKEN=$GITHUB_TOKEN ./ci/scripts/universal-package.sh"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Upload artifacts

.github/workflows/test-linux.yaml → .github/workflows/test.yaml

@@ -16,43 +16,39 @@ on:
         required: false
         default: false
 jobs:
-  ubuntu-build:
+  test:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - postgres: 16
-            os: ubuntu-22.04
-          - postgres: 15
-            os: ubuntu-22.04
-          - postgres: 14
-            os: ubuntu-22.04
-          - postgres: 13
-            os: ubuntu-22.04
-          - postgres: 12
-            os: ubuntu-22.04
-          - postgres: 11
-            os: ubuntu-22.04
+        os: [ubuntu-22.04, macos-13]
+        postgres: [11, 12, 13, 14, 15, 16]
     steps:
       - uses: actions/checkout@v3
         with:
           submodules: "recursive"
       - name: Build
         id: build
-        run: sudo su -c "PG_VERSION=$PG_VERSION USE_SOURCE=1 GITHUB_OUTPUT=$GITHUB_OUTPUT ENABLE_COVERAGE=1 ./ci/scripts/build-linux.sh"
+        run: sudo sh -c "PG_VERSION=$PG_VERSION USE_SOURCE=1 GITHUB_OUTPUT=$GITHUB_OUTPUT ENABLE_COVERAGE=1 ./ci/scripts/build.sh"
         env:
           PG_VERSION: ${{ matrix.postgres }}
           BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
       # Enable tmate debugging of manually-triggered workflows if the input option was provided
       - name: Setup tmate session
         uses: mxschmitt/action-tmate@v3
         if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
-      - name: Run tests
-        id: test
-        run: sudo su postgres -c "PG_VERSION=$PG_VERSION ./ci/scripts/run-tests.sh"
+      - name: Run tests linux
+        id: test-linux
+        run: sudo su postgres -c "PG_VERSION=$PG_VERSION ./ci/scripts/run-tests-linux.sh"
         env:
           PG_VERSION: ${{ matrix.postgres }}
+        if: ${{ startsWith(matrix.os, 'ubuntu') }}
+      - name: Run tests mac
+        id: test-mac
+        run: ./ci/scripts/run-tests-mac.sh
+        env:
+          PG_VERSION: ${{ matrix.postgres }}
+        if: ${{ startsWith(matrix.os, 'mac') }}
       - name: Upload Postgres logs
         if: failure()
         uses: actions/upload-artifact@v3
@@ -65,7 +61,7 @@ jobs:
         uses: codecov/codecov-action@v3
         env:
           codecov_token: ${{ secrets.CODECOV_TOKEN }}
-        if: ${{ env.codecov_token != '' && matrix.postgres == 15 }} # for now run only on once
+        if: ${{ env.codecov_token != '' && startsWith(matrix.os, 'ubuntu') && matrix.postgres == 15 }} # for now run only on once
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           name: codecov-lanterndb

ci/scripts/build-docker.sh

@@ -1,11 +1,7 @@
 #!/bin/bash
 
 get_cmake_flags(){
- # TODO:: remove after test
  echo "-DUSEARCH_NO_MARCH_NATIVE=ON"
- # if [[ $ARCH == *"arm"* ]]; then
- #   echo "-DUSEARCH_NO_MARCH_NATIVE=ON"
- # fi
 }
 
 export DEBIAN_FRONTEND=noninteractive

ci/scripts/build-linux.sh

@@ -1,14 +1,6 @@
 #!/bin/bash
 set -e
 
-function setup_environment() {
-  export BRANCH=${BRANCH_NAME:-dev}
-  export POSTGRES_USER=postgres
-  export DEBIAN_FRONTEND=noninteractive
-  export PG_VERSION=${PG_VERSION:-15}
-  export GITHUB_OUTPUT=${GITHUB_OUTPUT:-/dev/null}
-}
-
 function setup_locale_and_install_packages() {
   echo "LC_ALL=en_US.UTF-8" > /etc/environment
   echo "en_US.UTF-8 UTF-8" > /etc/locale.gen
@@ -22,60 +14,15 @@ function setup_locale_and_install_packages() {
 
 function setup_postgres() {
   # Add postgresql apt repo
-  export ARCH=$(dpkg-architecture -q DEB_BUILD_ARCH)
   echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
   wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc |  apt-key add -
   # Install postgres and dev files for C headers
   apt update
   apt install -y postgresql-$PG_VERSION postgresql-server-dev-$PG_VERSION
-  # Install pgvector
-  pushd /tmp
-    PGVECTOR_VERSION=0.5.0
-    wget -O pgvector.tar.gz https://github.com/pgvector/pgvector/archive/refs/tags/v${PGVECTOR_VERSION}.tar.gz
-    tar xzf pgvector.tar.gz
-    pushd pgvector-${PGVECTOR_VERSION}
-      make && make install
-    popd
-  popd
   # Fix pg_config (sometimes it points to wrong version)
   rm -f /usr/bin/pg_config && ln -s /usr/lib/postgresql/$PG_VERSION/bin/pg_config /usr/bin/pg_config
 }
 
-function clone_or_use_source() {
-  if [ -z ${USE_SOURCE} ]; then
-    # Clone from git
-    cd /tmp
-    git clone --recursive https://github.com/lanterndata/lantern.git -b $BRANCH
-  else 
-    # Use already checkouted code
-    shopt -s dotglob
-    mkdir -p /tmp/lantern
-    cp -r ./* /tmp/lantern/
-  fi
-}
-
-function build_and_install() {
-  cd /tmp/lantern
-  mkdir build
-  cd build
-
-   # TODO:: remove after test
-  flags="-DUSEARCH_NO_MARCH_NATIVE=ON"
-  # Treat warnings as errors in CI/CD
-  flags+=" -DCMAKE_COMPILE_WARNING_AS_ERROR=ON"
-  # if [[ $ARCH == *"arm"* ]]; then
-  #   echo "-DUSEARCH_NO_MARCH_NATIVE=ON"
-  # fi
-  if [ -n "$ENABLE_COVERAGE" ]
-  then
-    flags="$flags -DCMAKE_C_COMPILER=/usr/bin/gcc -DCODECOVERAGE=ON"
-  fi
-
-  # Run cmake
-  cmake $flags ..
-  make install
-}
-
 function package_if_necessary() {
   if [ -n "$BUILD_PACKAGES" ]; then
     # Bundle debian packages
@@ -84,18 +31,13 @@ function package_if_necessary() {
     export EXT_VERSION=$(cmake --system-information | awk -F= '$1~/CMAKE_PROJECT_VERSION:STATIC/{print$2}') && \
     export PACKAGE_NAME=lantern-${EXT_VERSION}-postgres-${PG_VERSION}-${ARCH}.deb && \
 
-    echo "package_version=$EXT_VERSION" >> "$GITHUB_OUTPUT" && \
-    echo "package_name=$PACKAGE_NAME" >> "$GITHUB_OUTPUT" && \
-    echo "package_path=$(pwd)/$(ls *.deb | tr -d '\n')" >> "$GITHUB_OUTPUT"
+    echo "deb_package_name=$PACKAGE_NAME" >> "$GITHUB_OUTPUT" && \
+    echo "deb_package_path=$(pwd)/$(ls *.deb | tr -d '\n')" >> "$GITHUB_OUTPUT"
   fi
 }
 
-setup_environment
-setup_locale_and_install_packages
-setup_postgres
-clone_or_use_source
-build_and_install
-package_if_necessary
+function cleanup_environment() {
+  # Chown to postgres for running tests
+  chown -R postgres:postgres /tmp/lantern
+}
 
-# Chown to postgres for running tests
-chown -R postgres:postgres /tmp/lantern

ci/scripts/build-mac.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -e
+
+function setup_locale_and_install_packages() {
+  export PATH="/usr/local/opt/postgresql@${PG_VERSION}/bin:$PATH"
+  export C_INCLUDE_PATH=/usr/local/include
+  export CPLUS_INCLUDE_PATH=/usr/local/include
+}
+
+function setup_postgres() {
+  cmd="brew install postgresql@${PG_VERSION} clang-format"
+  if [[ $USER == "root" ]]
+  then
+    # Runner is github CI user
+    sh -c "sudo -u runner -i $cmd"
+  else
+    sh -c $cmd
+  fi
+}
+
+function package_if_necessary() {
+  :
+  # TODO make and publish homebrew formula
+}
+
+function cleanup_environment() {
+  # Add permission to for running tests from runner user
+  chmod -R 777 /tmp/lantern
+}
+

ci/scripts/build.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+set -e
+
+# This sets $ARCH and $PLATFORM env variables
+source "$(dirname "$0")/../../scripts/get_arch_and_platform.sh"
+
+if [[ $PLATFORM == "mac" ]]; then
+   BUILD_SCRIPT="build-mac.sh"
+elif [[ $PLATFORM == "linux" ]]; then
+   BUILD_SCRIPT="build-linux.sh"
+else
+   echo "Invalid target use one of [mac, linux]"
+   exit 1
+fi
+
+function setup_environment() {
+  export BRANCH=${BRANCH_NAME:-dev}
+  export POSTGRES_USER=postgres
+  export DEBIAN_FRONTEND=noninteractive
+  export PG_VERSION=${PG_VERSION:-15}
+  export GITHUB_OUTPUT=${GITHUB_OUTPUT:-/dev/null}
+}
+
+function clone_or_use_source() {
+  if [ -z ${USE_SOURCE} ]; then
+    # Clone from git
+    cd /tmp
+    git clone --recursive https://github.com/lanterndata/lantern.git -b $BRANCH
+  else 
+    # Use already checkouted code
+    shopt -s dotglob
+    rm -rf /tmp/lantern
+    mkdir -p /tmp/lantern
+    cp -r -P ./* /tmp/lantern/
+  fi
+}
+
+function install_external_dependencies() {
+  # Install pgvector
+  pushd /tmp
+    PGVECTOR_VERSION=0.5.0
+    wget -O pgvector.tar.gz https://github.com/pgvector/pgvector/archive/refs/tags/v${PGVECTOR_VERSION}.tar.gz
+    tar xzf pgvector.tar.gz
+    pushd pgvector-${PGVECTOR_VERSION}
+      make && make install
+    popd
+  popd
+}
+
+function build_and_install() {
+  cd /tmp/lantern
+  mkdir build
+  cd build
+
+  flags="-DUSEARCH_NO_MARCH_NATIVE=ON"
+
+  # Treat warnings as errors in CI/CD
+  flags+=" -DCMAKE_COMPILE_WARNING_AS_ERROR=ON"
+
+  if [ -n "$ENABLE_COVERAGE" ]
+  then
+    flags="$flags -DCMAKE_C_COMPILER=/usr/bin/gcc -DCODECOVERAGE=ON"
+  fi
+
+  # Run cmake
+  cmake $flags ..
+  make install
+}
+
+# Source platform specific build script
+source "$(dirname "$0")/${BUILD_SCRIPT}"
+
+setup_environment
+setup_locale_and_install_packages
+setup_postgres
+install_external_dependencies
+clone_or_use_source
+build_and_install
+package_if_necessary
+cleanup_environment

ci/scripts/package-archive.sh

@@ -1,9 +1,6 @@
 #!/bin/bash
 
-if [ -z "$GITHUB_OUTPUT" ]
-then
-  export GITHUB_OUTPUT=/dev/null
-fi
+GITHUB_OUTPUT=${GITHUB_OUTPUT:-/dev/null}
 
 cd /tmp/lantern/build && make archive
 cat /tmp/gh-output.txt >> $GITHUB_OUTPUT && rm -rf /tmp/gh-output

ci/scripts/run-benchmarks.sh

@@ -18,9 +18,9 @@ wait_for_pg(){
 export WORKDIR=/tmp/lantern
 export PG_VERSION=15
 export GITHUB_OUTPUT=/dev/null
-export PGDATA=/etc/postgresql/$PG_VERSION/main/
+export PGDATA=/etc/postgresql/$PG_VERSION/main
 
-echo "port = 5432" >> $PGDATA/postgresql.conf
+echo "port = 5432" >> ${PGDATA}/postgresql.conf
 # Enable auth without password
 echo "local   all             all                                     trust" >  $PGDATA/pg_hba.conf
 echo "host    all             all             127.0.0.1/32            trust" >>  $PGDATA/pg_hba.conf