Merge branch 'dev' into azure-ad

.github/workflows/append-release-cmake.yml

@@ -0,0 +1,80 @@
+name: Append Release CMake
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'Ref to be used as release'
+        default: 'latest'
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Ref to be used as release'
+        default: 'latest'
+        required: true
+        type: string
+
+jobs:
+  generate_cmake_files:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout TileDB
+        uses: actions/checkout@v3
+
+      - name: Make release and output directories
+        run: |
+          mkdir release output
+
+      - name: Github release data
+        id: release_data
+        uses: KevinRohn/[email protected]
+        with:
+          # repository: 'TileDB-Inc/TileDB'
+          version: ${{ inputs.ref }}
+          asset-file: '*.zip,*.tar.gz'
+          asset-output: './release/'
+
+      - name: Render template
+        run: |
+          PATTERN="tiledb-([^-]+)-([^-]+)(-noavx2)?-([^-]+).(tar.gz|zip)$"
+          RELLIST="output/releases.csv"
+          MODULE="output/DownloadPrebuiltTileDB.cmake"
+          cp cmake/inputs/DownloadPrebuiltTileDB.cmake $MODULE
+          echo "platform,url,sha256" > $RELLIST
+          
+          for FILE in $(ls release)
+          do
+                  if [[ $FILE =~ $PATTERN ]]
+                  then
+                          OS=${BASH_REMATCH[1]^^}
+                          ARCH=${BASH_REMATCH[2]^^}
+                          NOAVX2=${BASH_REMATCH[3]^^}
+                          PLATFORM=${OS}-${ARCH}${NOAVX2}
+          
+                          URL="${{ github.server_url }}/${{ github.repository }}/releases/download/${{ inputs.ref }}/$FILE"
+                          HASH=$(cat release/$FILE.sha256 | cut -d \t -f 1)
+          
+                          echo "${PLATFORM},${URL},${HASH}" >> $RELLIST
+                  fi
+          done
+          
+          SOURCE_FILE_NAME=$(ls release/tiledb-source-*.tar.gz)
+          URL_TILEDB_SOURCE="${{ github.server_url }}/${{ github.repository }}/releases/download/${{ inputs.ref }}/$(basename $SOURCE_FILE_NAME)"
+          HASH_TILEDB_SOURCE=$(cat $SOURCE_FILE_NAME.sha256 | cut -d \t -f 1)
+          
+          echo "source,${URL_TILEDB_SOURCE},${HASH_TILEDB_SOURCE}" >> $RELLIST
+          
+          HASH=$(sha256sum $RELLIST | cut -d " " -f 1)
+          echo $HASH > $RELLIST.sha256
+          
+          cat $RELLIST
+
+      - name: Upload template to release
+        uses: svenstaro/upload-release-action@v2
+        with:
+          file: output/*
+          tag: ${{ steps.release_data.outputs.tag_name }}
+          overwrite: true
+          file_glob: true

.github/workflows/build-docs.yml

@@ -28,6 +28,8 @@ jobs:
     runs-on: ubuntu-22.04
     timeout-minutes: 90
     name: Build Docs
+    env:
+      VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite'
     steps:
       - uses: actions/checkout@v3
       - name: 'Print env'
@@ -41,6 +43,13 @@ jobs:
           printenv
         shell: bash
 
+      - name: Set environment variables for vcpkg binary caching
+        uses: actions/github-script@v6
+        with:
+          script: |
+            core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+
       - name: Set up Python
         uses: actions/setup-python@v4
 

.github/workflows/build-ubuntu20.04-backwards-compatibility.yml

@@ -59,7 +59,7 @@ jobs:
           - ubuntu-20.04
         # Note: v2_1_0 arrays were never created so its currently skipped
         # Note: This matrix is used to set the value of TILEDB_COMPATIBILITY_VERSION
-        tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1"]
+        tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1", "v2_20_1"]
     timeout-minutes: 30
     name: ${{ matrix.tiledb_version }}
     steps:

.github/workflows/build-windows.yml

@@ -60,8 +60,22 @@ jobs:
       TILEDB_ARROW_TESTS: ${{ matrix.TILEDB_ARROW_TESTS }}
       TILEDB_WEBP: ${{ matrix.TILEDB_WEBP }}
       TILEDB_CMAKE_BUILD_TYPE: 'Release'
+      # On windows-2019 we are using the Visual Studio generator, which is multi-config and places the build artifacts in a subdirectory
+      CONFIG_PATH_FIXUP: ${{ matrix.os == 'windows-2019' && 'Release' || '' }}
       VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite'
     steps:
+      # By default Visual Studio chooses the earliest installed toolset version
+      # for the main build and vcpkg chooses the latest. Force it to use the
+      # latest (14.39 currently).
+      - name: Setup MSVC toolset (VS 2022)
+        uses: TheMrMilchmann/setup-msvc-dev@v3
+        if: matrix.os == 'windows-2022'
+        with:
+          arch: x64
+          toolset: 14.39
+      - name: Install Ninja (VS 2022)
+        uses: seanmiddleditch/gha-setup-ninja@v4
+        if: matrix.os == 'windows-2022'
       - name: 'tiledb env prep'
         run: |
           $env:BUILD_BUILDDIRECTORY = $env:GITHUB_WORKSPACE.replace("TileDB\TileDB","tdbbd") # 't'ile'db' 'b'uild 'd'ir
@@ -141,7 +155,7 @@ jobs:
           # allow double-checking path
           cmd /c "echo $PATH"
 
-          $bootstrapOptions = $env:TILEDB_BASE_BOOTSTRAP_OPTIONS
+          $bootstrapOptions = $env:TILEDB_BASE_BOOTSTRAP_OPTIONS + " -CMakeGenerator ${{ matrix.os == 'windows-2022' && 'Ninja' || '`"Visual Studio 16 2019`"' }}"
           if ($env:TILEDB_S3 -eq "ON") {
             $bootstrapOptions = "-EnableS3 " + $bootstrapOptions
           }
@@ -239,9 +253,7 @@ jobs:
           }
 
           # CMake exits with non-0 status if there are any warnings during the build, so
-          cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target tiledb_unit --config $CMakeBuildType -- /verbosity:minimal
-          cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target tiledb_regression --config $CMakeBuildType -- /verbosity:minimal
-          cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target all_link_complete --config $CMakeBuildType -- /verbosity:minimal
+          cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target tiledb_unit unit_vfs tiledb_regression all_link_complete --config $CMakeBuildType
 
           if ($env:TILEDB_AZURE -eq "ON") {
             if($env.TILEDB_USE_CUSTOM_NODE_JS) {
@@ -272,15 +284,23 @@ jobs:
 
           # Actually run tests
 
-          $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\$CMakeBuildType\tiledb_unit.exe -d=yes"
+          $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\$env:CONFIG_PATH_FIXUP\tiledb_unit.exe -d=yes"
           Write-Host "cmds: '$cmds'"
           Invoke-Expression $cmds
           if ($LastExitCode -ne 0) {
              Write-Host "Tests failed. tiledb_unit exit status: " $LastExitCocde
              $host.SetShouldExit($LastExitCode)
           }
+          
+          $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\tiledb\sm\filesystem\test\$env:CONFIG_PATH_FIXUP\unit_vfs -d=yes"
+          Write-Host "cmds: '$cmds'"
+          Invoke-Expression $cmds
+          if ($LastExitCode -ne 0) {
+             Write-Host "Tests failed. tiledb_vfs exit status: " $LastExitCocde
+             $host.SetShouldExit($LastExitCode)
+          }
 
-          $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\ci\$CMakeBuildType\test_assert.exe -d=yes"
+          $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\ci\$env:CONFIG_PATH_FIXUP\test_assert.exe -d=yes"
           Invoke-Expression $cmds
           if ($LastExitCode -ne 0) {
              Write-Host "Tests failed. test_assert exit status: " $LastExitCocde
@@ -302,7 +322,7 @@ jobs:
 
           $TestAppDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api")
           $TestAppDataDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api\test_app_data")
-          Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api\$CMakeBuildType") -Filter *.exe |
+          Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api\$env:CONFIG_PATH_FIXUP\") -Filter *.exe |
           Foreach-Object {
             try {
               Set-Location -path $TestAppDir
@@ -335,7 +355,7 @@ jobs:
 
           $TestAppDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api")
           $TestAppDataDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api\test_app_data")
-          Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api\$CMakeBuildType") -Filter *.exe |
+          Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api\$env:CONFIG_PATH_FIXUP\") -Filter *.exe |
           Foreach-Object {
             try {
               Set-Location -path $TestAppDir
@@ -380,12 +400,12 @@ jobs:
           cd build
 
           # Build zip artifact
-          cmake -A X64 -DCMAKE_PREFIX_PATH="$env:BUILD_BUILDDIRECTORY\dist;$env:BUILD_BUILDDIRECTORY\vcpkg_installed\x64-windows" ..
+          cmake ${{ matrix.os != 'windows-2019' && '-G Ninja' || '' }} -DCMAKE_BUILD_TYPE="$CMakeBuildType" -DCMAKE_PREFIX_PATH="$env:BUILD_BUILDDIRECTORY\dist;$env:BUILD_BUILDDIRECTORY\vcpkg_installed\x64-windows" ..
 
           cmake --build . --config $CMakeBuildType -v
 
-          #.\$CMakeBuildType\ExampleExe.exe
-          $cmd = ".\$CMakeBuildType\ExampleExe.exe"
+          #.\$env:CONFIG_PATH_FIXUP\ExampleExe.exe
+          $cmd = ".\$env:CONFIG_PATH_FIXUP\ExampleExe.exe"
           Write-Host "cmd: '$cmd'"
           Invoke-Expression $cmd
 

.github/workflows/ci-linux_mac.yml

@@ -63,7 +63,7 @@ env:
   CC: ${{ inputs.matrix_compiler_cc }}
   CFLAGS: ${{ inputs.matrix_compiler_cflags }}
   CXXFLAGS: ${{ inputs.matrix_compiler_cxxflags }}
-  bootstrap_args: "--enable-ccache ${{ inputs.bootstrap_args }} ${{ inputs.asan && '--enable-sanitizer=address' || '' }}"
+  bootstrap_args: "--enable-ccache --vcpkg-base-triplet=x64-${{ startsWith(inputs.matrix_image, 'ubuntu-') && 'linux' || 'osx' }} ${{ inputs.bootstrap_args }} ${{ inputs.asan && '--enable-sanitizer=address' || '' }}"
   VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite'
   SCCACHE_GHA_ENABLED: "true"
 
@@ -191,6 +191,7 @@ jobs:
           ./tiledb/test/regression/tiledb_regression -d yes
           ./tiledb/test/ci/test_assert -d yes
           ./tiledb/test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
+          ./tiledb/tiledb/sm/filesystem/test/unit_vfs -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
 
           ###################################################
           # Stop helper processes, if applicable

.github/workflows/ci-rest.yml

@@ -1,29 +1,35 @@
 name: REST CI
 
 on:
-  - workflow_call
-  - workflow_dispatch
+  workflow_call:
+  workflow_dispatch:
+
+  push:
+    branches:
+      - dev
+      - release-*
+      - refs/tags/*
 
 jobs:
   rest-ci:
     runs-on: ubuntu-latest
 
     steps:
       # For easy access to lookup dispatched CI job.
-      - name: Print URL for TileDB-REST-CI actions
-        run: echo https://github.com/TileDB-Inc/TileDB-REST-CI/actions
+      - name: Print URL for REST CI actions
+        run: echo https://github.com/TileDB-Inc/TileDB-Internal/actions
 
       # If this workflow fails on the remote repository, this CI job will also fail.
-      - name: Workflow dispatch to TileDB-REST-CI
+      - name: Workflow dispatch to REST CI
         id: trigger-step
         uses: aurelien-baudet/workflow-dispatch@v2
         env:
           TILEDB_REST_CI_PAT: ${{ secrets.TILEDB_REST_CI_PAT }}
         # Skip if no PAT is set (e.g. for PRs from forks).
         if: env.TILEDB_REST_CI_PAT != null
         with:
-          repo: TileDB-Inc/TileDB-REST-CI
-          # Trigger workflow on TileDB-REST-CI at this ref.
+          repo: TileDB-Inc/TileDB-Internal
+          # Trigger workflow on TileDB-Internal at this ref.
           ref: "main"
           workflow: full-ci.yml
           token: ${{ secrets.TILEDB_REST_CI_PAT }}

.github/workflows/full-ci.yml

@@ -101,7 +101,7 @@ jobs:
       matrix_compiler_cc: 'gcc-10'
       matrix_compiler_cxx: 'g++-10'
       timeout: 120
-      bootstrap_args: '--enable-serialization --vcpkg-base-triplet=x64-linux'
+      bootstrap_args: '--enable-serialization'
       asan: true
 
   ci10:
@@ -138,10 +138,6 @@ jobs:
   ci_docker:
     uses: ./.github/workflows/build-dockerfile.yml
 
-  ci_rest:
-    uses: ./.github/workflows/ci-rest.yml
-    secrets: inherit
-
   # dummy job for branch protection check
   full_ci_passed:
     needs: [