diff --git a/.github/workflows/ci-linux_mac.yml b/.github/workflows/ci-linux_mac.yml index a02ac7ed0af..94c0178e6ea 100644 --- a/.github/workflows/ci-linux_mac.yml +++ b/.github/workflows/ci-linux_mac.yml @@ -14,7 +14,7 @@ on: type: string bootstrap_args: default: '' - required: true + required: false type: string matrix_image: default: '' @@ -28,6 +28,14 @@ on: default: 'g++' required: false type: string + matrix_compiler_cflags: + default: '' + required: false + type: string + matrix_compiler_cxxflags: + default: '' + required: false + type: string timeout: default: 90 description: 'Job timeout (minutes)' @@ -38,13 +46,23 @@ on: description: 'Enable Address Sanitizer' required: false type: boolean + manylinux: + default: false + description: 'Enable manylinux builds' + required: false + type: boolean env: BACKWARDS_COMPATIBILITY_ARRAYS: OFF TILEDB_CI_BACKEND: ${{ inputs.ci_backend }} TILEDB_CI_OS: runner.os + # Installing Python does not work on manylinux. + TILEDB_ARROW_TESTS: ${{ !inputs.manylinux && 'ON' || 'OFF' }} + TILEDB_MANYLINUX: ${{ !inputs.manylinux && 'ON' || 'OFF' }} CXX: ${{ inputs.matrix_compiler_cxx }} CC: ${{ inputs.matrix_compiler_cc }} + CFLAGS: ${{ inputs.matrix_compiler_cflags }} + CXXFLAGS: ${{ inputs.matrix_compiler_cxxflags }} bootstrap_args: "--enable-ccache ${{ inputs.bootstrap_args }} ${{ inputs.asan && '--enable-sanitizer=address' || '' }}" VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' SCCACHE_GHA_ENABLED: "true" @@ -56,6 +74,7 @@ jobs: os: - ${{ inputs.matrix_image }} runs-on: ${{matrix.os}} + container: ${{inputs.manylinux && 'quay.io/pypa/manylinux2014_x86_64' || ''}} if: ${{ startsWith(github.ref , 'refs/tags') != true && startsWith(github.ref , 'build-') != true }} timeout-minutes: ${{ inputs.timeout || 90 }} @@ -85,14 +104,23 @@ jobs: run: ./scripts/ci/posix/prelim.sh shell: bash + - name: Install manylinux prerequisites + if: inputs.manylinux + run: | + set -e pipefail + yum install -y redhat-lsb-core centos-release-scl devtoolset-7 + echo "source /opt/rh/devtoolset-7/enable" >> ~/.bashrc + # Need this for virtualenv and arrow tests if enabled - name: 'Install Python' uses: actions/setup-python@v4 + if: ${{ !inputs.manylinux }} with: python-version: '3.8' cache: 'pip' - name: 'Set up Python dependencies' + if: ${{ !inputs.manylinux }} run: | set -e pipefail python -m pip install --upgrade pip virtualenv diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml index 13ebef77d48..a710d9761fb 100644 --- a/.github/workflows/full-ci.yml +++ b/.github/workflows/full-ci.yml @@ -112,6 +112,17 @@ jobs: timeout: 120 bootstrap_args: '--enable-azure' + ci_manylinux: + uses: ./.github/workflows/ci-linux_mac.yml + with: + ci_backend: MANYLINUX + matrix_image: ubuntu-20.04 + matrix_compiler_cflags: "-lrt" + matrix_compiler_cxxflags: "-lrt" + timeout: 120 + bootstrap_args: '--enable-serialization' + manylinux: true + ci_msvc: uses: ./.github/workflows/build-windows.yml @@ -145,6 +156,7 @@ jobs: ci8, ci9, ci10, + ci_manylinux, ci_msvc, backward_compatibility, standalone diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b0d6841e369..236ae26cc83 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,17 +5,6 @@ trigger: - release-* - refs/tags/* - build- -pr: - branches: - include: - - '*' # must quote since "*" is a YAML reserved character; we want a string - paths: - exclude: - - '.github/workflows/quarto-render.yml' - - '_quarto.yml' - - 'quarto-materials/*' - - '**/.md' - - 'doc/source/conf.py' variables: - name: MANYLINUX_IMAGE @@ -24,25 +13,6 @@ variables: value: ghcr.io/ihnorton/tiledb-manylinux2014_x86_64:2023-04-02 stages: - - stage: CI - condition: and(not(startsWith(variables['Build.SourceBranch'], 'refs/tags')), not(startsWith(variables['Build.SourceBranchName'], 'build-'))) - variables: - BACKWARDS_COMPATIBILITY_ARRAYS: OFF - jobs: - - job: linux_manylinux - pool: { vmImage: $(imageName) } - container: ${{ variables.MANYLINUX_IMAGE }} - variables: - imageName: 'ubuntu-20.04' - TILEDB_SERIALIZATION: ON - CXX: g++ - CC: gcc - CXXFLAGS: "-lrt" - CFLAGS: "-lrt" - USE_MANYLINUX: ON - steps: - - template: scripts/azure-linux_mac.yml - - stage: Build_Release condition: or(or(or(startsWith(variables['Build.SourceBranch'], 'refs/tags'), eq(variables['Build.SourceBranchName'], 'dev')), startsWith(variables['Build.SourceBranchName'], 'release-')), startsWith(variables['Build.SourceBranchName'], 'build-')) variables: diff --git a/scripts/azure-linux_mac.yml b/scripts/azure-linux_mac.yml deleted file mode 100755 index 57e06eff32e..00000000000 --- a/scripts/azure-linux_mac.yml +++ /dev/null @@ -1,283 +0,0 @@ -steps: -- task: NodeTool@0 # Needed for compatibility with manylinux image. - inputs: - versionSpec: '16.x' - displayName: 'Install Node.js 16' - -- bash: | - echo "'uname -s' is:" - echo "uname: " $(uname) - echo "uname -m: " $(uname -m) - echo "uname -r:" $(uname -r) - echo "uname -s: " $(uname -s) - echo "uname -v: " $(uname -v) - printenv - displayName: 'Print env' - -# Need this for virtualenv and arrow tests if enabled -- task: UsePythonVersion@0 - inputs: - versionSpec: '3.8' - condition: eq(variables['TILEDB_ARROW_TESTS'], 'ON') - -- bash: | - set -e pipefail - python -m pip install --upgrade pip virtualenv - if [[ "$TILEDB_ARROW_TESTS" == "ON" ]]; then - pip install pyarrow pybind11 numpy - fi - condition: eq(variables['TILEDB_ARROW_TESTS'], 'ON') - -- bash: | - #https://stackoverflow.com/questions/65278351/no-core-dump-file-generated-after-segmentation-fault-in-macos-big-sur - sysctl -a | grep core - - #https://developer.apple.com/forums/thread/53023?answerId=158378022#158378022 - #sudo chmod 1775 /cores - #https://developer.apple.com/forums/thread/127503?answerId=401103022#401103022 - #so seems after change to 1777, the 'runner' attempt did core dump, while the sudo version did not... - sudo chmod 1777 /cores - sudo chown $(whoami) /cores - - sudo ls -l / - - ps -A - ls -l /cores - condition: and(eq(variables['Agent.OS'], 'Darwin'), startsWith(variables['imageName'], 'macOS-')) - displayName: 'prep macOS for "runner" core dumps' - - -- bash: | - brew install autoconf automake - condition: eq(variables['Agent.OS'], 'Darwin') - -- bash: | - # enable core dumps - ulimit -c # should output 0 if disabled - ulimit -c unlimited # Enable core dumps to be captured (must be in same run block) - ulimit -c # should output 'unlimited' now - - # Azure sets "SYSTEM=build" for unknown reasons, which breaks the OpenSSL configure script - # - openssl configure uses ENV{SYSTEM} if available: - # https://github.com/openssl/openssl/blob/6d745d740d37d680ff696486218b650512bbbbc6/config#L56 - # - error description: - # https://developercommunity.visualstudio.com/content/problem/602584/openssl-build-error-when-using-pipelines.htm - unset SYSTEM - - # azure bash does not treat intermediate failure as error - # https://github.com/Microsoft/azure-pipelines-yaml/issues/135 - set -xe pipefail - - git config --global user.name 'Azure Pipeline' - git config --global user.email 'no-reply@tiledb.io' - - if [[ "$BACKWARDS_COMPATIBILITY_ARRAYS" == "ON" ]]; then - git clone https://github.com/TileDB-Inc/TileDB-Unit-Test-Arrays.git --branch 2.5.0 test/inputs/arrays/read_compatibility_test - fi - # displayName: 'Clone Unit-Test-Arrays' - - # - bash: | - # Start HDFS server if enabled - if [[ "$TILEDB_HDFS" == "ON" ]]; then - # - ssh to localhost is required for HDFS launch... - # - /home/vsts has permissions g+w and is owned by user 'docker' - # for VSTS purposes, so disable ssh strictness - sudo sed -i "s/StrictModes\ yes/StrictModes\ no/g" /etc/ssh/sshd_config - - source scripts/install-hadoop.sh - source scripts/run-hadoop.sh - fi - - # Start minio server if S3 is enabled - if [[ "$TILEDB_S3" == "ON" ]]; then - source scripts/install-minio.sh; - source scripts/run-minio.sh; - fi - - # Start Azurite if Azure is enabled - if [[ "$TILEDB_AZURE" == "ON" ]]; then - source scripts/install-azurite.sh; - source scripts/run-azurite.sh; - fi - - # Start GCS Emulator if GCS is enabled - if [[ "$TILEDB_GCS" == "ON" ]]; then - source scripts/install-gcs-emu.sh; - source scripts/run-gcs-emu.sh; - fi - - # Set up arguments for bootstrap.sh - bootstrap_args="--enable=verbose"; - - # Enable TILEDB_STATIC by default - [ "$TILEDB_STATIC" ] || TILEDB_STATIC=ON - if [[ "$TILEDB_STATIC" == "ON" ]]; then - bootstrap_args="${bootstrap_args} --enable-static-tiledb"; - fi - if [[ "$TILEDB_HDFS" == "ON" ]]; then - bootstrap_args="${bootstrap_args} --enable-hdfs"; - fi; - if [[ "$TILEDB_S3" == "ON" ]]; then - bootstrap_args="${bootstrap_args} --enable-s3"; - fi; - if [[ "$TILEDB_AZURE" == "ON" ]]; then - bootstrap_args="${bootstrap_args} --enable-azure"; - fi; - if [[ "$TILEDB_GCS" == "ON" ]]; then - bootstrap_args="${bootstrap_args} --enable-gcs"; - fi; - if [[ "$TILEDB_TOOLS" == "ON" ]]; then - bootstrap_args="${bootstrap_args} --enable-tools"; - fi - if [[ "$TILEDB_DEBUG" == "ON" ]]; then - bootstrap_args="${bootstrap_args} --enable-debug"; - fi - if [[ "$TILEDB_CI_ASAN" == "ON" ]]; then - # Add address sanitizer flag if necessary - bootstrap_args="${bootstrap_args} --enable-sanitizer=address --enable-debug"; - fi - if [[ "$TILEDB_CI_TSAN" == "ON" ]]; then - # Add thread sanitizer flag if necessary - bootstrap_args="${bootstrap_args} --enable-sanitizer=thread --enable-debug"; - fi - if [[ "$TILEDB_SERIALIZATION" == "ON" ]]; then - # Add serialization flag if necessary - bootstrap_args="${bootstrap_args} --enable-serialization"; - fi - if [[ "$TILEDB_FORCE_BUILD_DEPS" == "ON" ]]; then - # Add superbuild flag - bootstrap_args="${bootstrap_args} --force-build-all-deps"; - fi - if [[ "$TILEDB_WEBP" == "OFF" ]]; then - bootstrap_args="${bootstrap_args} --disable-webp"; - fi; - if [[ "$AGENT_OS" == "Darwin" ]]; then - # We want to be able to print a stack trace when a core dump occurs - sudo chmod 1777 /cores - # release symbols closer to 'release' than debug, and will hopefully have 'enough' - # context for reasonable stack. 'debug' (lack of) optimization less likely to - # to produce problems that might be encountered by actual 'release' build. - bootstrap_args="${bootstrap_args} --enable-release-symbols"; - fi - - # displayName: 'Install dependencies' - - mkdir -p $BUILD_REPOSITORY_LOCALPATH/build - cd $BUILD_REPOSITORY_LOCALPATH/build - - # Configure and build TileDB - echo "Bootstrapping with '$bootstrap_args'" - $BUILD_REPOSITORY_LOCALPATH/bootstrap $bootstrap_args - - make -j4 - make examples -j4 - make -C tiledb install - - #- bash: | - cd $BUILD_REPOSITORY_LOCALPATH/build - ls -la - - if [[ ( "$AGENT_OS" == "Linux" && "$TILEDB_S3" == "ON" ) ]]; then - # make sure docker is still running... - printenv - docker ps -a - fi - - make -j4 -C tiledb tiledb_unit - make -j4 -C tiledb tiledb_regression - - if [[ "$TILEDB_CI_ASAN" == "ON" ]]; then - export ASAN_OPTIONS=detect_leaks=0 LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.5 - fi - - if [[ "$AGENT_OS" == "Darwin" && "$TILEDB_GCS" == "ON" ]]; then - # GCS unit tests are temporarily unsupported on CI for MACOS. Fake success with - # this echo. - echo "##vso[task.setvariable variable=TILEDB_CI_SUCCESS]1" - else - # run directly the executable, cmake catches the segfault and blocks - # the core dump - ./tiledb/test/tiledb_unit -d yes - ./tiledb/test/regression/tiledb_regression -d yes - fi - - # Kill the running Minio server, OSX only because Linux runs it within - # docker. - if [[ ( "$AGENT_OS" == "Darwin" && "$TILEDB_S3" == "ON" ) ]]; then - kill -9 $MINIO_PID - fi - - # Kill the running Azurite server - if [[ "$TILEDB_AZURE" == "ON" ]]; then - kill -9 $AZURITE_PID - fi - - # Kill the running GCS emulator server Linux only because OSX does not - # run the emulator - if [[ "$AGENT_OS" != "Darwin" && "$TILEDB_GCS" == "ON" ]]; then - kill -9 $GCS_PID - fi - - displayName: 'Build and test libtiledb' - -- bash: | - set -x - ls -la /cores - cntfiles=0 - for f in $(find /cores -name 'core.*'); - do - echo "stack trace for $f" - lldb -c $f --batch -o 'bt all' -o 'image list' -o 're r -a' -o 'di -F intel -f -m' -o 'quit' - (( cntfiles = cntfiles + 1 )) - done; - #mv whatever we can, possible we won't get all due to permissions - #and archiving may have chance of failing due to size constraints in runner VM - mkdir -p $BUILD_REPOSITORY_LOCALPATH/build/core - mv -v /cores/core.* $BUILD_REPOSITORY_LOCALPATH/build/core/ - ls -la $BUILD_REPOSITORY_LOCALPATH/build/core - if [[ cntfiles -ne 0 ]] ; then - export TDB_COREFILE_CNT=$cntfiles - fi - condition: and(failed(), eq(variables['Agent.OS'], 'Darwin')) # only run this job if the build step failed - displayName: 'Print stack trace' - -- task: ArchiveFiles@2 - inputs: - rootFolderOrFile: '$(Build.Repository.LocalPath)' - includeRootFolder: false - archiveType: 'tar' - tarCompression: 'gz' - archiveFile: '$(Build.ArtifactStagingDirectory)/$(ARTIFACT_OS)-build-dir.tar.gz' - replaceExistingArchive: true - verbose: true - condition: and(failed(), eq(variables['Agent.OS'], 'Darwin')) # only run this job if the build step failed - -- task: PublishBuildArtifacts@1 - inputs: - pathToPublish: '$(Build.ArtifactStagingDirectory)/$(ARTIFACT_OS)-build-dir.tar.gz' - artifactName: 'build-dirs' - condition: and(failed(), eq(variables['Agent.OS'], 'Darwin')) # only run this job if the build step failed - -- bash: | - # tiledb_unit is configured to set a job-level variable TILEDB_CI_SUCCESS=1 - # following the test run. If this variable is not set, the build should fail. - # see https://github.com/TileDB-Inc/TileDB/pull/1400 (5f0623f4d3) - if [[ "$TILEDB_CI_SUCCESS" -ne 1 ]]; then - exit 1; - fi - displayName: 'Test status check' - - -- bash: | - set -e pipefail - # Display log files if the build failed - echo "Dumping log files for failed build" - echo "----------------------------------" - for f in $(find $BUILD_REPOSITORY_LOCALPATH/build -name *.log); - do echo "------" - echo $f - echo "======" - cat $f - done; - condition: failed() # only run this job if the build step failed - displayName: "Print log files (failed build only)" diff --git a/test/src/unit-capi-query.cc b/test/src/unit-capi-query.cc index 816ffce7190..8a0b3c1c9cc 100644 --- a/test/src/unit-capi-query.cc +++ b/test/src/unit-capi-query.cc @@ -33,6 +33,7 @@ #include #include +#include #include #include #include @@ -713,6 +714,10 @@ TEST_CASE_METHOD( // DenyWriteAccess is not supported on Windows. if constexpr (tiledb::platform::is_os_windows) return; + // The test fails on Manylinux. Skip it. + char* manylinux_var = getenv("TILEDB_MANYLINUX"); + if (manylinux_var && strlen(manylinux_var) > 0) + return; SupportedFsLocal local_fs; std::string temp_dir = local_fs.temp_dir(); std::string array_name = temp_dir + "write_failure";