From fe1e96b9671892358444369386d5dbb8ea3a243d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 28 Dec 2022 14:33:31 -0600 Subject: [PATCH 1/5] [ci] speed up conda env creation in Windows jobs (fixes #5647) --- .appveyor.yml | 2 + .ci/test_windows.ps1 | 13 +- .github/workflows/cuda.yml | 118 ------- .github/workflows/python_package.yml | 82 ----- .github/workflows/r_package.yml | 257 --------------- .vsts-ci.yml | 460 ++++++++++++++------------- 6 files changed, 245 insertions(+), 687 deletions(-) delete mode 100644 .github/workflows/cuda.yml delete mode 100644 .github/workflows/python_package.yml delete mode 100644 .github/workflows/r_package.yml diff --git a/.appveyor.yml b/.appveyor.yml index ba85fac817ab..2d279b0f33e3 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -36,6 +36,8 @@ install: build: false test_script: + - conda config --remove channels defaults + - conda config --add channels nodefaults - conda config --add channels conda-forge - conda config --set channel_priority strict - conda init powershell diff --git a/.ci/test_windows.ps1 b/.ci/test_windows.ps1 index 6c0ffb8249f0..36806cef3ed7 100644 --- a/.ci/test_windows.ps1 +++ b/.ci/test_windows.ps1 @@ -30,7 +30,18 @@ conda init powershell conda activate conda config --set always_yes yes --set changeps1 no conda update -q -y conda -conda create -q -y -n $env:CONDA_ENV "python=$env:PYTHON_VERSION[build=*cpython]" ; Check-Output $? +conda create -q -y -n $env:CONDA_ENV ` + cloudpickle ` + joblib ` + matplotlib ` + numpy ` + pandas ` + psutil ` + pytest ` + "python=$env:PYTHON_VERSION[build=*cpython]" ` + python-graphviz ` + scikit-learn ` + scipy ; Check-Output $? if ($env:TASK -ne "bdist") { conda activate $env:CONDA_ENV } diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml deleted file mode 100644 index 50a3aa71dde8..000000000000 --- a/.github/workflows/cuda.yml +++ /dev/null @@ -1,118 +0,0 @@ -name: CUDA Version - -on: - push: - branches: - - master - pull_request: - branches: - - master - - release/* - -env: - github_actions: 'true' - os_name: linux - conda_env: test-env - -jobs: - test: - name: ${{ matrix.task }} ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) - runs-on: [self-hosted, linux] - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - method: source - compiler: gcc - python_version: "3.8" - cuda_version: "11.7.1" - task: cuda - - method: pip - compiler: clang - python_version: "3.9" - cuda_version: "10.0" - task: cuda - - method: wheel - compiler: gcc - python_version: "3.10" - cuda_version: "9.0" - task: cuda - - method: source - compiler: gcc - python_version: "3.8" - cuda_version: "11.7.1" - task: cuda_exp - - method: pip - compiler: clang - python_version: "3.9" - cuda_version: "10.0" - task: cuda_exp - steps: - - name: Setup or update software on host machine - run: | - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - apt-transport-https \ - ca-certificates \ - curl \ - git \ - gnupg-agent \ - lsb-release \ - software-properties-common - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y - curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - - curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - containerd.io \ - docker-ce \ - docker-ce-cli \ - nvidia-docker2 - sudo chmod a+rw /var/run/docker.sock - sudo systemctl restart docker - - name: Remove old folder with repository - run: sudo rm -rf $GITHUB_WORKSPACE - - name: Checkout repository - uses: actions/checkout@v1 - with: - fetch-depth: 5 - submodules: true - - name: Setup and run tests - run: | - export ROOT_DOCKER_FOLDER=/LightGBM - cat > docker.env < docker-script.sh <> tests.log 2>&1 || exit_code=-1 - cat ./tests.log - exit ${exit_code} - test-r-debian-clang: - name: r-package (debian, R-devel, clang) - timeout-minutes: 60 - runs-on: ubuntu-latest - container: rhub/debian-clang-devel - steps: - - name: Install Git before checkout - shell: bash - run: | - apt-get update --allow-releaseinfo-change - apt-get install --no-install-recommends -y git - - name: Trust git cloning LightGBM - run: | - git config --global --add safe.directory "${GITHUB_WORKSPACE}" - - name: Checkout repository - uses: actions/checkout@v3 - with: - fetch-depth: 5 - submodules: true - - name: Install packages and run tests - shell: bash - run: | - export PATH=/opt/R-devel/bin/:${PATH} - Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'RhpcBLASctl', 'rmarkdown', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" - sh build-cran-package.sh - R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit -1 - if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then - echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check" - exit -1 - fi - all-r-package-jobs-successful: - if: always() - runs-on: ubuntu-latest - needs: [test, test-r-sanitizers, test-r-debian-clang] - steps: - - name: Note that all tests succeeded - uses: re-actors/alls-green@v1.2.2 - with: - jobs: ${{ toJSON(needs) }} diff --git a/.vsts-ci.yml b/.vsts-ci.yml index a01e994e0a91..08448a9a43f9 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -26,235 +26,235 @@ resources: - container: rbase image: wch1/r-debug jobs: -########################################### -- job: Linux -########################################### - variables: - COMPILER: gcc - SETUP_CONDA: 'false' - OS_NAME: 'linux' - PRODUCES_ARTIFACTS: 'true' - pool: sh-ubuntu - container: linux-artifact-builder - strategy: - matrix: - regular: - TASK: regular - PYTHON_VERSION: '3.9' - sdist: - TASK: sdist - PYTHON_VERSION: '3.7' - bdist: - TASK: bdist - PYTHON_VERSION: '3.8' - inference: - TASK: if-else - mpi_source: - TASK: mpi - METHOD: source - PYTHON_VERSION: '3.8' - gpu_source: - TASK: gpu - METHOD: source - swig: - TASK: swig - steps: - - script: | - echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" - echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" - echo "##vso[task.prependpath]/usr/lib64/openmpi/bin" - echo "##vso[task.prependpath]$CONDA/bin" - displayName: 'Set variables' - - script: | - echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt' - displayName: 'Add commit hash to artifacts archive' - - bash: $(Build.SourcesDirectory)/.ci/setup.sh - displayName: Setup - - bash: $(Build.SourcesDirectory)/.ci/test.sh - displayName: Test - - task: PublishBuildArtifacts@1 - condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: PackageAssets - artifactType: container -########################################### -- job: Linux_latest -########################################### - variables: - COMPILER: clang - DEBIAN_FRONTEND: 'noninteractive' - IN_UBUNTU_BASE_CONTAINER: 'true' - OS_NAME: 'linux' - SETUP_CONDA: 'true' - pool: sh-ubuntu - container: ubuntu-latest - strategy: - matrix: - regular: - TASK: regular - sdist: - TASK: sdist - bdist: - TASK: bdist - PYTHON_VERSION: '3.8' - inference: - TASK: if-else - mpi_source: - TASK: mpi - METHOD: source - mpi_pip: - TASK: mpi - METHOD: pip - PYTHON_VERSION: '3.9' - mpi_wheel: - TASK: mpi - METHOD: wheel - PYTHON_VERSION: '3.7' - gpu_source: - TASK: gpu - METHOD: source - PYTHON_VERSION: '3.9' - gpu_pip: - TASK: gpu - METHOD: pip - PYTHON_VERSION: '3.8' - gpu_wheel: - TASK: gpu - METHOD: wheel - PYTHON_VERSION: '3.7' - cpp_tests: - TASK: cpp-tests - METHOD: with-sanitizers - steps: - - script: | - echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" - echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" - CONDA=$HOME/miniforge - echo "##vso[task.setvariable variable=CONDA]$CONDA" - echo "##vso[task.prependpath]$CONDA/bin" - displayName: 'Set variables' - # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 - - script: | - /tmp/docker exec -t -u 0 ci-container \ - sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" - displayName: 'Install sudo' - - bash: $(Build.SourcesDirectory)/.ci/setup.sh - displayName: Setup - - bash: $(Build.SourcesDirectory)/.ci/test.sh - displayName: Test -########################################### -- job: QEMU_multiarch -########################################### - variables: - COMPILER: gcc - OS_NAME: 'linux' - PRODUCES_ARTIFACTS: 'true' - pool: - vmImage: ubuntu-22.04 - timeoutInMinutes: 180 - strategy: - matrix: - bdist: - TASK: bdist - ARCH: aarch64 - steps: - - script: | - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - binfmt-support \ - qemu \ - qemu-user \ - qemu-user-static - displayName: 'Install QEMU' - - script: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - displayName: 'Enable Docker multi-architecture support' - - script: | - export ROOT_DOCKER_FOLDER=/LightGBM - cat > docker.env < docker-script.sh < '$(Build.ArtifactStagingDirectory)/commit.txt' +# displayName: 'Add commit hash to artifacts archive' +# - bash: $(Build.SourcesDirectory)/.ci/setup.sh +# displayName: Setup +# - bash: $(Build.SourcesDirectory)/.ci/test.sh +# displayName: Test +# - task: PublishBuildArtifacts@1 +# condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) +# inputs: +# pathtoPublish: '$(Build.ArtifactStagingDirectory)' +# artifactName: PackageAssets +# artifactType: container +############################################ +#- job: Linux_latest +############################################ +# variables: +# COMPILER: clang +# DEBIAN_FRONTEND: 'noninteractive' +# IN_UBUNTU_BASE_CONTAINER: 'true' +# OS_NAME: 'linux' +# SETUP_CONDA: 'true' +# pool: sh-ubuntu +# container: ubuntu-latest +# strategy: +# matrix: +# regular: +# TASK: regular +# sdist: +# TASK: sdist +# bdist: +# TASK: bdist +# PYTHON_VERSION: '3.8' +# inference: +# TASK: if-else +# mpi_source: +# TASK: mpi +# METHOD: source +# mpi_pip: +# TASK: mpi +# METHOD: pip +# PYTHON_VERSION: '3.9' +# mpi_wheel: +# TASK: mpi +# METHOD: wheel +# PYTHON_VERSION: '3.7' +# gpu_source: +# TASK: gpu +# METHOD: source +# PYTHON_VERSION: '3.9' +# gpu_pip: +# TASK: gpu +# METHOD: pip +# PYTHON_VERSION: '3.8' +# gpu_wheel: +# TASK: gpu +# METHOD: wheel +# PYTHON_VERSION: '3.7' +# cpp_tests: +# TASK: cpp-tests +# METHOD: with-sanitizers +# steps: +# - script: | +# echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" +# echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" +# CONDA=$HOME/miniforge +# echo "##vso[task.setvariable variable=CONDA]$CONDA" +# echo "##vso[task.prependpath]$CONDA/bin" +# displayName: 'Set variables' +# # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 +# - script: | +# /tmp/docker exec -t -u 0 ci-container \ +# sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" +# displayName: 'Install sudo' +# - bash: $(Build.SourcesDirectory)/.ci/setup.sh +# displayName: Setup +# - bash: $(Build.SourcesDirectory)/.ci/test.sh +# displayName: Test +############################################ +#- job: QEMU_multiarch +############################################ +# variables: +# COMPILER: gcc +# OS_NAME: 'linux' +# PRODUCES_ARTIFACTS: 'true' +# pool: +# vmImage: ubuntu-22.04 +# timeoutInMinutes: 180 +# strategy: +# matrix: +# bdist: +# TASK: bdist +# ARCH: aarch64 +# steps: +# - script: | +# sudo apt-get update +# sudo apt-get install --no-install-recommends -y \ +# binfmt-support \ +# qemu \ +# qemu-user \ +# qemu-user-static +# displayName: 'Install QEMU' +# - script: | +# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes +# displayName: 'Enable Docker multi-architecture support' +# - script: | +# export ROOT_DOCKER_FOLDER=/LightGBM +# cat > docker.env < docker-script.sh < Date: Wed, 28 Dec 2022 14:35:31 -0600 Subject: [PATCH 2/5] remove second 'conda install' --- .ci/test_windows.ps1 | 3 --- 1 file changed, 3 deletions(-) diff --git a/.ci/test_windows.ps1 b/.ci/test_windows.ps1 index 36806cef3ed7..a953050b42c6 100644 --- a/.ci/test_windows.ps1 +++ b/.ci/test_windows.ps1 @@ -61,9 +61,6 @@ if ($env:TASK -eq "swig") { Exit 0 } -# re-including python=version[build=*cpython] to ensure that conda doesn't fall back to pypy -conda install -q -y -n $env:CONDA_ENV cloudpickle joblib matplotlib numpy pandas psutil pytest "python=$env:PYTHON_VERSION[build=*cpython]" python-graphviz scikit-learn scipy ; Check-Output $? - if ($env:TASK -eq "regular") { mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build cmake -A x64 .. ; cmake --build . --target ALL_BUILD --config Release ; Check-Output $? From f352764c377a7a20b6be896cb5896a5378eac5c0 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 28 Dec 2022 14:40:16 -0600 Subject: [PATCH 3/5] comment out more stuff --- .vsts-ci.yml | 204 +++++++++++++++++++++++++-------------------------- 1 file changed, 102 insertions(+), 102 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 08448a9a43f9..a6fbd05fed79 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -296,105 +296,105 @@ jobs: pathtoPublish: '$(Build.ArtifactStagingDirectory)' artifactName: PackageAssets artifactType: container -########################################### -- job: R_artifact -########################################### - condition: not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')) - pool: - vmImage: 'ubuntu-22.04' - container: rbase - steps: - - script: | - LGB_VER=$(head -n 1 VERSION.txt | sed "s/rc/-/g") - R_LIB_PATH=~/Rlib - export R_LIBS=${R_LIB_PATH} - mkdir -p ${R_LIB_PATH} - RDscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'RhpcBLASctl', 'rmarkdown'), lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" || exit -1 - sh build-cran-package.sh --r-executable=RD || exit -1 - mv lightgbm_${LGB_VER}.tar.gz $(Build.ArtifactStagingDirectory)/lightgbm-${LGB_VER}-r-cran.tar.gz - displayName: 'Build CRAN R-package' - - task: PublishBuildArtifacts@1 - condition: succeeded() - inputs: - pathtoPublish: $(Build.ArtifactStagingDirectory) - artifactName: R-package - artifactType: container - -########################################### -- job: Package -########################################### - dependsOn: - - Linux - - Linux_latest - - QEMU_multiarch - - macOS - - Windows - - R_artifact - condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) - pool: - vmImage: 'ubuntu-22.04' - steps: - # Create archives with complete source code included (with git submodules) - - task: ArchiveFiles@2 - displayName: Create zip archive - condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) - inputs: - rootFolderOrFile: $(Build.SourcesDirectory) - includeRootFolder: false - archiveType: zip - archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_zip.zip' - replaceExistingArchive: true - - task: ArchiveFiles@2 - displayName: Create tar.gz archive - condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) - inputs: - rootFolderOrFile: $(Build.SourcesDirectory) - includeRootFolder: false - archiveType: tar - tarCompression: gz - archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_tar_gz.tar.gz' - replaceExistingArchive: true - # Download all agent packages from all previous phases - - task: DownloadBuildArtifacts@0 - displayName: Download package assets - inputs: - artifactName: PackageAssets - downloadPath: $(Build.SourcesDirectory)/binaries - - task: DownloadBuildArtifacts@0 - displayName: Download R-package - condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) - inputs: - artifactName: R-package - downloadPath: $(Build.SourcesDirectory)/R - - script: | - python "$(Build.SourcesDirectory)/.nuget/create_nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets" - displayName: 'Create NuGet configuration files' - - task: NuGetCommand@2 - inputs: - command: pack - packagesToPack: '$(Build.SourcesDirectory)/.nuget/*.nuspec' - packDestination: '$(Build.ArtifactStagingDirectory)/nuget' - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget' - artifactName: NuGet - artifactType: container - - task: GitHubRelease@0 - displayName: 'Create GitHub Release' - condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) - inputs: - gitHubConnection: guolinke - repositoryName: '$(Build.Repository.Name)' - action: 'create' - target: '$(Build.SourceVersion)' - tagSource: 'auto' - title: '$(Build.SourceBranchName)' - assets: | - $(Build.SourcesDirectory)/binaries/PackageAssets/* - $(Build.SourcesDirectory)/R/R-package/* - $(Build.ArtifactStagingDirectory)/nuget/*.nupkg - $(Build.ArtifactStagingDirectory)/archives/* - assetUploadMode: 'delete' - isDraft: true - isPreRelease: false - addChangeLog: false +############################################ +#- job: R_artifact +############################################ +# condition: not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')) +# pool: +# vmImage: 'ubuntu-22.04' +# container: rbase +# steps: +# - script: | +# LGB_VER=$(head -n 1 VERSION.txt | sed "s/rc/-/g") +# R_LIB_PATH=~/Rlib +# export R_LIBS=${R_LIB_PATH} +# mkdir -p ${R_LIB_PATH} +# RDscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'RhpcBLASctl', 'rmarkdown'), lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" || exit -1 +# sh build-cran-package.sh --r-executable=RD || exit -1 +# mv lightgbm_${LGB_VER}.tar.gz $(Build.ArtifactStagingDirectory)/lightgbm-${LGB_VER}-r-cran.tar.gz +# displayName: 'Build CRAN R-package' +# - task: PublishBuildArtifacts@1 +# condition: succeeded() +# inputs: +# pathtoPublish: $(Build.ArtifactStagingDirectory) +# artifactName: R-package +# artifactType: container +# +############################################ +#- job: Package +############################################ +# dependsOn: +# - Linux +# - Linux_latest +# - QEMU_multiarch +# - macOS +# - Windows +# - R_artifact +# condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) +# pool: +# vmImage: 'ubuntu-22.04' +# steps: +# # Create archives with complete source code included (with git submodules) +# - task: ArchiveFiles@2 +# displayName: Create zip archive +# condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) +# inputs: +# rootFolderOrFile: $(Build.SourcesDirectory) +# includeRootFolder: false +# archiveType: zip +# archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_zip.zip' +# replaceExistingArchive: true +# - task: ArchiveFiles@2 +# displayName: Create tar.gz archive +# condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) +# inputs: +# rootFolderOrFile: $(Build.SourcesDirectory) +# includeRootFolder: false +# archiveType: tar +# tarCompression: gz +# archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_tar_gz.tar.gz' +# replaceExistingArchive: true +# # Download all agent packages from all previous phases +# - task: DownloadBuildArtifacts@0 +# displayName: Download package assets +# inputs: +# artifactName: PackageAssets +# downloadPath: $(Build.SourcesDirectory)/binaries +# - task: DownloadBuildArtifacts@0 +# displayName: Download R-package +# condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) +# inputs: +# artifactName: R-package +# downloadPath: $(Build.SourcesDirectory)/R +# - script: | +# python "$(Build.SourcesDirectory)/.nuget/create_nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets" +# displayName: 'Create NuGet configuration files' +# - task: NuGetCommand@2 +# inputs: +# command: pack +# packagesToPack: '$(Build.SourcesDirectory)/.nuget/*.nuspec' +# packDestination: '$(Build.ArtifactStagingDirectory)/nuget' +# - task: PublishBuildArtifacts@1 +# inputs: +# pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget' +# artifactName: NuGet +# artifactType: container +# - task: GitHubRelease@0 +# displayName: 'Create GitHub Release' +# condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) +# inputs: +# gitHubConnection: guolinke +# repositoryName: '$(Build.Repository.Name)' +# action: 'create' +# target: '$(Build.SourceVersion)' +# tagSource: 'auto' +# title: '$(Build.SourceBranchName)' +# assets: | +# $(Build.SourcesDirectory)/binaries/PackageAssets/* +# $(Build.SourcesDirectory)/R/R-package/* +# $(Build.ArtifactStagingDirectory)/nuget/*.nupkg +# $(Build.ArtifactStagingDirectory)/archives/* +# assetUploadMode: 'delete' +# isDraft: true +# isPreRelease: false +# addChangeLog: false From 8f22877aa82d70cd130b513114d9804bce80e098 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 28 Dec 2022 15:09:43 -0600 Subject: [PATCH 4/5] bring all the other CI jobs back --- .ci/test_windows.ps1 | 1 + .github/workflows/cuda.yml | 118 +++++ .github/workflows/python_package.yml | 82 ++++ .github/workflows/r_package.yml | 257 +++++++++++ .vsts-ci.yml | 662 +++++++++++++-------------- 5 files changed, 789 insertions(+), 331 deletions(-) create mode 100644 .github/workflows/cuda.yml create mode 100644 .github/workflows/python_package.yml create mode 100644 .github/workflows/r_package.yml diff --git a/.ci/test_windows.ps1 b/.ci/test_windows.ps1 index a953050b42c6..cc284d6f1515 100644 --- a/.ci/test_windows.ps1 +++ b/.ci/test_windows.ps1 @@ -42,6 +42,7 @@ conda create -q -y -n $env:CONDA_ENV ` python-graphviz ` scikit-learn ` scipy ; Check-Output $? + if ($env:TASK -ne "bdist") { conda activate $env:CONDA_ENV } diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml new file mode 100644 index 000000000000..50a3aa71dde8 --- /dev/null +++ b/.github/workflows/cuda.yml @@ -0,0 +1,118 @@ +name: CUDA Version + +on: + push: + branches: + - master + pull_request: + branches: + - master + - release/* + +env: + github_actions: 'true' + os_name: linux + conda_env: test-env + +jobs: + test: + name: ${{ matrix.task }} ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) + runs-on: [self-hosted, linux] + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - method: source + compiler: gcc + python_version: "3.8" + cuda_version: "11.7.1" + task: cuda + - method: pip + compiler: clang + python_version: "3.9" + cuda_version: "10.0" + task: cuda + - method: wheel + compiler: gcc + python_version: "3.10" + cuda_version: "9.0" + task: cuda + - method: source + compiler: gcc + python_version: "3.8" + cuda_version: "11.7.1" + task: cuda_exp + - method: pip + compiler: clang + python_version: "3.9" + cuda_version: "10.0" + task: cuda_exp + steps: + - name: Setup or update software on host machine + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + apt-transport-https \ + ca-certificates \ + curl \ + git \ + gnupg-agent \ + lsb-release \ + software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y + curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - + curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + containerd.io \ + docker-ce \ + docker-ce-cli \ + nvidia-docker2 + sudo chmod a+rw /var/run/docker.sock + sudo systemctl restart docker + - name: Remove old folder with repository + run: sudo rm -rf $GITHUB_WORKSPACE + - name: Checkout repository + uses: actions/checkout@v1 + with: + fetch-depth: 5 + submodules: true + - name: Setup and run tests + run: | + export ROOT_DOCKER_FOLDER=/LightGBM + cat > docker.env < docker-script.sh <> tests.log 2>&1 || exit_code=-1 + cat ./tests.log + exit ${exit_code} + test-r-debian-clang: + name: r-package (debian, R-devel, clang) + timeout-minutes: 60 + runs-on: ubuntu-latest + container: rhub/debian-clang-devel + steps: + - name: Install Git before checkout + shell: bash + run: | + apt-get update --allow-releaseinfo-change + apt-get install --no-install-recommends -y git + - name: Trust git cloning LightGBM + run: | + git config --global --add safe.directory "${GITHUB_WORKSPACE}" + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 5 + submodules: true + - name: Install packages and run tests + shell: bash + run: | + export PATH=/opt/R-devel/bin/:${PATH} + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'RhpcBLASctl', 'rmarkdown', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" + sh build-cran-package.sh + R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit -1 + if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then + echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check" + exit -1 + fi + all-r-package-jobs-successful: + if: always() + runs-on: ubuntu-latest + needs: [test, test-r-sanitizers, test-r-debian-clang] + steps: + - name: Note that all tests succeeded + uses: re-actors/alls-green@v1.2.2 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.vsts-ci.yml b/.vsts-ci.yml index a6fbd05fed79..af0d63322004 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -26,235 +26,235 @@ resources: - container: rbase image: wch1/r-debug jobs: -############################################ -#- job: Linux -############################################ -# variables: -# COMPILER: gcc -# SETUP_CONDA: 'false' -# OS_NAME: 'linux' -# PRODUCES_ARTIFACTS: 'true' -# pool: sh-ubuntu -# container: linux-artifact-builder -# strategy: -# matrix: -# regular: -# TASK: regular -# PYTHON_VERSION: '3.9' -# sdist: -# TASK: sdist -# PYTHON_VERSION: '3.7' -# bdist: -# TASK: bdist -# PYTHON_VERSION: '3.8' -# inference: -# TASK: if-else -# mpi_source: -# TASK: mpi -# METHOD: source -# PYTHON_VERSION: '3.8' -# gpu_source: -# TASK: gpu -# METHOD: source -# swig: -# TASK: swig -# steps: -# - script: | -# echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" -# echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" -# echo "##vso[task.prependpath]/usr/lib64/openmpi/bin" -# echo "##vso[task.prependpath]$CONDA/bin" -# displayName: 'Set variables' -# - script: | -# echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt' -# displayName: 'Add commit hash to artifacts archive' -# - bash: $(Build.SourcesDirectory)/.ci/setup.sh -# displayName: Setup -# - bash: $(Build.SourcesDirectory)/.ci/test.sh -# displayName: Test -# - task: PublishBuildArtifacts@1 -# condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) -# inputs: -# pathtoPublish: '$(Build.ArtifactStagingDirectory)' -# artifactName: PackageAssets -# artifactType: container -############################################ -#- job: Linux_latest -############################################ -# variables: -# COMPILER: clang -# DEBIAN_FRONTEND: 'noninteractive' -# IN_UBUNTU_BASE_CONTAINER: 'true' -# OS_NAME: 'linux' -# SETUP_CONDA: 'true' -# pool: sh-ubuntu -# container: ubuntu-latest -# strategy: -# matrix: -# regular: -# TASK: regular -# sdist: -# TASK: sdist -# bdist: -# TASK: bdist -# PYTHON_VERSION: '3.8' -# inference: -# TASK: if-else -# mpi_source: -# TASK: mpi -# METHOD: source -# mpi_pip: -# TASK: mpi -# METHOD: pip -# PYTHON_VERSION: '3.9' -# mpi_wheel: -# TASK: mpi -# METHOD: wheel -# PYTHON_VERSION: '3.7' -# gpu_source: -# TASK: gpu -# METHOD: source -# PYTHON_VERSION: '3.9' -# gpu_pip: -# TASK: gpu -# METHOD: pip -# PYTHON_VERSION: '3.8' -# gpu_wheel: -# TASK: gpu -# METHOD: wheel -# PYTHON_VERSION: '3.7' -# cpp_tests: -# TASK: cpp-tests -# METHOD: with-sanitizers -# steps: -# - script: | -# echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" -# echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" -# CONDA=$HOME/miniforge -# echo "##vso[task.setvariable variable=CONDA]$CONDA" -# echo "##vso[task.prependpath]$CONDA/bin" -# displayName: 'Set variables' -# # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 -# - script: | -# /tmp/docker exec -t -u 0 ci-container \ -# sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" -# displayName: 'Install sudo' -# - bash: $(Build.SourcesDirectory)/.ci/setup.sh -# displayName: Setup -# - bash: $(Build.SourcesDirectory)/.ci/test.sh -# displayName: Test -############################################ -#- job: QEMU_multiarch -############################################ -# variables: -# COMPILER: gcc -# OS_NAME: 'linux' -# PRODUCES_ARTIFACTS: 'true' -# pool: -# vmImage: ubuntu-22.04 -# timeoutInMinutes: 180 -# strategy: -# matrix: -# bdist: -# TASK: bdist -# ARCH: aarch64 -# steps: -# - script: | -# sudo apt-get update -# sudo apt-get install --no-install-recommends -y \ -# binfmt-support \ -# qemu \ -# qemu-user \ -# qemu-user-static -# displayName: 'Install QEMU' -# - script: | -# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -# displayName: 'Enable Docker multi-architecture support' -# - script: | -# export ROOT_DOCKER_FOLDER=/LightGBM -# cat > docker.env < docker-script.sh < '$(Build.ArtifactStagingDirectory)/commit.txt' + displayName: 'Add commit hash to artifacts archive' + - bash: $(Build.SourcesDirectory)/.ci/setup.sh + displayName: Setup + - bash: $(Build.SourcesDirectory)/.ci/test.sh + displayName: Test + - task: PublishBuildArtifacts@1 + condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: PackageAssets + artifactType: container +########################################### +- job: Linux_latest +########################################### + variables: + COMPILER: clang + DEBIAN_FRONTEND: 'noninteractive' + IN_UBUNTU_BASE_CONTAINER: 'true' + OS_NAME: 'linux' + SETUP_CONDA: 'true' + pool: sh-ubuntu + container: ubuntu-latest + strategy: + matrix: + regular: + TASK: regular + sdist: + TASK: sdist + bdist: + TASK: bdist + PYTHON_VERSION: '3.8' + inference: + TASK: if-else + mpi_source: + TASK: mpi + METHOD: source + mpi_pip: + TASK: mpi + METHOD: pip + PYTHON_VERSION: '3.9' + mpi_wheel: + TASK: mpi + METHOD: wheel + PYTHON_VERSION: '3.7' + gpu_source: + TASK: gpu + METHOD: source + PYTHON_VERSION: '3.9' + gpu_pip: + TASK: gpu + METHOD: pip + PYTHON_VERSION: '3.8' + gpu_wheel: + TASK: gpu + METHOD: wheel + PYTHON_VERSION: '3.7' + cpp_tests: + TASK: cpp-tests + METHOD: with-sanitizers + steps: + - script: | + echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" + echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" + CONDA=$HOME/miniforge + echo "##vso[task.setvariable variable=CONDA]$CONDA" + echo "##vso[task.prependpath]$CONDA/bin" + displayName: 'Set variables' + # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 + - script: | + /tmp/docker exec -t -u 0 ci-container \ + sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" + displayName: 'Install sudo' + - bash: $(Build.SourcesDirectory)/.ci/setup.sh + displayName: Setup + - bash: $(Build.SourcesDirectory)/.ci/test.sh + displayName: Test +########################################### +- job: QEMU_multiarch +########################################### + variables: + COMPILER: gcc + OS_NAME: 'linux' + PRODUCES_ARTIFACTS: 'true' + pool: + vmImage: ubuntu-22.04 + timeoutInMinutes: 180 + strategy: + matrix: + bdist: + TASK: bdist + ARCH: aarch64 + steps: + - script: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + binfmt-support \ + qemu \ + qemu-user \ + qemu-user-static + displayName: 'Install QEMU' + - script: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + displayName: 'Enable Docker multi-architecture support' + - script: | + export ROOT_DOCKER_FOLDER=/LightGBM + cat > docker.env < docker-script.sh < Date: Wed, 28 Dec 2022 15:59:00 -0600 Subject: [PATCH 5/5] run 'swig' task before any conda stuff --- .ci/test_windows.ps1 | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.ci/test_windows.ps1 b/.ci/test_windows.ps1 index cc284d6f1515..79b35faaff10 100644 --- a/.ci/test_windows.ps1 +++ b/.ci/test_windows.ps1 @@ -25,6 +25,21 @@ if ($env:TASK -eq "cpp-tests") { Exit 0 } +if ($env:TASK -eq "swig") { + $env:JAVA_HOME = $env:JAVA_HOME_8_X64 # there is pre-installed Eclipse Temurin 8 somewhere + $ProgressPreference = "SilentlyContinue" # progress bar bug extremely slows down download speed + Invoke-WebRequest -Uri "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/swigwin-4.0.2.zip" -OutFile $env:BUILD_SOURCESDIRECTORY/swig/swigwin.zip -UserAgent "NativeHost" + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory("$env:BUILD_SOURCESDIRECTORY/swig/swigwin.zip", "$env:BUILD_SOURCESDIRECTORY/swig") + $env:PATH = "$env:BUILD_SOURCESDIRECTORY/swig/swigwin-4.0.2;" + $env:PATH + mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build + cmake -A x64 -DUSE_SWIG=ON .. ; cmake --build . --target ALL_BUILD --config Release ; Check-Output $? + if ($env:AZURE -eq "true") { + cp $env:BUILD_SOURCESDIRECTORY/build/lightgbmlib.jar $env:BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_win.jar + } + Exit 0 +} + # setup for Python conda init powershell conda activate @@ -47,21 +62,6 @@ if ($env:TASK -ne "bdist") { conda activate $env:CONDA_ENV } -if ($env:TASK -eq "swig") { - $env:JAVA_HOME = $env:JAVA_HOME_8_X64 # there is pre-installed Eclipse Temurin 8 somewhere - $ProgressPreference = "SilentlyContinue" # progress bar bug extremely slows down download speed - Invoke-WebRequest -Uri "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/swigwin-4.0.2.zip" -OutFile $env:BUILD_SOURCESDIRECTORY/swig/swigwin.zip -UserAgent "NativeHost" - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory("$env:BUILD_SOURCESDIRECTORY/swig/swigwin.zip", "$env:BUILD_SOURCESDIRECTORY/swig") - $env:PATH = "$env:BUILD_SOURCESDIRECTORY/swig/swigwin-4.0.2;" + $env:PATH - mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build - cmake -A x64 -DUSE_SWIG=ON .. ; cmake --build . --target ALL_BUILD --config Release ; Check-Output $? - if ($env:AZURE -eq "true") { - cp $env:BUILD_SOURCESDIRECTORY/build/lightgbmlib.jar $env:BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_win.jar - } - Exit 0 -} - if ($env:TASK -eq "regular") { mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build cmake -A x64 .. ; cmake --build . --target ALL_BUILD --config Release ; Check-Output $?