From 55b219b5e8080020580e7ba7acc2223ff4f856c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 9 Apr 2021 18:52:12 +0200 Subject: [PATCH 01/30] Check if nightly build is needed and prepare version --- .github/workflows/nightly.yml | 602 ++++++++++++++++++++++++++++ tools/ci/nightly/package.json | 14 + tools/ci/nightly/preflight-check.js | 56 +++ 3 files changed, 672 insertions(+) create mode 100644 .github/workflows/nightly.yml create mode 100644 tools/ci/nightly/package.json create mode 100644 tools/ci/nightly/preflight-check.js diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 000000000000..fe323519a13d --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,602 @@ +# TODO we may consider merging this with `release.yml` as there is quite an overlap +name: Nightly Release CI + +on: + push: + schedule: + - cron: '0 4 * * 2-6' # 4am from Tuesday to Saturday (i.e. after every workday) + +env: + # Please ensure that this is in sync with graalVersion in build.sbt + graalVersion: 21.0.0.2 + # Please ensure that this is in sync with javaVersion in build.sbt + javaVersion: 11 + # Please ensure that this is in sync with project/build.properties + sbtVersion: 1.4.9 + # Please ensure that this is in sync with rustVersion in build.sbt + rustToolchain: nightly-2019-11-04 + +jobs: + preflight-check: + name: Check If The Build Should Proceed + runs-on: ubuntu-18.04 + timeout-minutes: 10 + # That true here is for debugging on staging + if: > + ${{ github.event_name == 'schedule' || contains(github.event.head_commit.message,'[release: nightly]') || true }} + outputs: + proceed: + description: Whether the nightly build should proceed. + value: ${{ steps.check-assumptions.outputs.proceed }} + nightly-version: + description: The version string to use for the current build. + value: ${{ steps.compute-version.outputs.nightly-version }} + steps: + - id: check-assumptions + name: Check If The Build Should Proceed + shell: bash + # The script below sets an output 'proceed' to true or false depending on whether the nightly build should proceed. + # Nightly builds are skipped if no new changes are present since the last one. + run: | + cd tools/ci/nightly/ + node preflight-check.js ${{ github.sha }} + - name: Checkout + if: ${{ steps.check-assumptions.outputs.proceed == 'true' }} + uses: actions/checkout@v2 + with: + path: repo + - name: Compute Version + id: compute-version + if: ${{ steps.check-assumptions.outputs.proceed == 'true' }} + shell: bash + run: | + VERSION=$(cat build.sbt | grep 'val ensoVersion' | sed 's/.*"\([^)]*\)".*/\1/').$(date --iso-8601) + echo ::set-output name=proceed::$VERSION + + # This job should be kept up-to-date with scala.yml#build (but keep the added version check) + build: + name: Build + runs-on: ${{ matrix.os }} + timeout-minutes: 90 + strategy: + matrix: + os: [macOS-latest, ubuntu-18.04, windows-latest] + fail-fast: true + needs: + - preflight-check + if: ${{ needs.preflight-check.outputs.proceed == 'true' }} + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + path: repo + - name: Debug Print commit message + shell: bash + run: | + echo ${{ github.event.head_commit.message }} + - name: Enable Developer Command Prompt (Windows) + uses: ilammy/msvc-dev-cmd@v1.5.0 + - name: Disable TCP/UDP Offloading (macOS) + if: runner.os == 'macOS' + shell: bash + run: | + sudo sysctl -w net.link.generic.system.hwcksum_tx=0 + sudo sysctl -w net.link.generic.system.hwcksum_rx=0 + - name: Disable TCP/UDP Offloading (Linux) + if: runner.os == 'Linux' + shell: bash + run: sudo ethtool -K eth0 tx off rx off + - name: Disable TCP/UDP Offloading (Windows) + if: runner.os == 'Windows' + shell: powershell + run: > + Disable-NetAdapterChecksumOffload -Name * -TcpIPv4 -UdpIPv4 -TcpIPv6 + -UdpIPv6 + - name: Install Rust + uses: actions-rs/toolchain@v1.0.6 + with: + toolchain: ${{ env.rustToolchain }} + override: true + - name: Setup conda + uses: s-weigand/setup-conda@v1.0.5 + with: + update-conda: true + conda-channels: anaconda, conda-forge + - name: Setup Conda Environment on Windows + if: runner.os == 'Windows' + run: | + conda create --name enso + conda init powershell + - name: Activate Conda Environment on Windows + if: runner.os == 'Windows' + run: conda activate enso + - name: Install FlatBuffers Compiler + run: conda install flatbuffers=1.12.0 + - name: Setup GraalVM Environment + uses: ayltai/setup-graalvm@v1 + with: + graalvm-version: ${{ env.graalVersion }} + java-version: ${{ env.javaVersion }} + native-image: true + - name: Set Up SBT + shell: bash + run: | + curl -fsSL -o sbt.tgz https://github.com/sbt/sbt/releases/download/v${{env.sbtVersion}}/sbt-${{env.sbtVersion}}.tgz + tar -xzf sbt.tgz + echo $GITHUB_WORKSPACE/sbt/bin/ >> $GITHUB_PATH + + # Caches + - name: Cache SBT + uses: actions/cache@v2 + with: + path: | + ~/.sbt + ~/.ivy2/cache + ~/.cache + key: ${{ runner.os }}-sbt-${{ hashFiles('**build.sbt') }} + restore-keys: ${{ runner.os }}-sbt- + + # Bootstrap + - name: Prepare Environment + shell: bash + run: | + echo "ENSO_RELEASE_MODE=true" >> $GITHUB_ENV + echo "ENSO_VERSION=${{ needs.preflight-check.outputs.nightly-version }}" >> $GITHUB_ENV + - name: Update the Version Number to the Nightly + shell: bash + run: | + sed -i "s/val ensoVersion.*= \".*\"/val ensoVersion = \"$ENSO_VERSION\"/" build.sbt + - name: Bootstrap the Project + working-directory: repo + shell: bash + run: | + sleep 1 + sbt --no-colors bootstrap + + # Verify Legal Review + - name: Verify Packages + if: runner.os != 'Windows' # TODO [RW] CRLF handling in licenses task + working-directory: repo + shell: bash + run: | + sleep 1 + sbt --no-colors verifyLicensePackages + + # Prepare distributions + - name: Build the Launcher Native Image + working-directory: repo + shell: bash + run: | + sleep 1 + sbt --no-colors launcher/buildNativeImage + + - name: Prepare Distribution Version (Unix) + working-directory: repo + if: runner.os != 'Windows' + shell: bash + run: | + chmod +x enso + DIST_VERSION=$(./enso version --json --only-launcher | jq -r '.version') + echo "DIST_VERSION=$DIST_VERSION" >> $GITHUB_ENV + + - name: Prepare Distribution Version (Windows) + working-directory: repo + if: runner.os == 'Windows' + shell: bash + run: | + DIST_VERSION=$(./enso.exe version --json --only-launcher | jq -r '.version') + echo "DIST_VERSION=$DIST_VERSION" >> $GITHUB_ENV + + # Currently the only architecture supported by Github runners is amd64 + - name: Prepare Distribution Environment + working-directory: repo + shell: bash + run: > + GRAAL_VERSION=$(echo ${{ env.graalVersion }}) DIST_OS=$(echo + ${{runner.os }} | awk '{print tolower($0)}') bash + tools/ci/prepare-distribution-env.sh + + - name: Prepare Launcher Distribution + working-directory: repo + shell: bash + run: | + sleep 1 + sbt buildLauncherDistribution + - name: Prepare Engine Distribution + working-directory: repo + shell: bash + run: | + sleep 1 + sbt buildEngineDistribution + - name: Prepare Project Manager Distribution + working-directory: repo + shell: bash + run: | + sleep 1 + sbt buildProjectManagerDistribution + - name: Prepare GraalVM Distribution + working-directory: repo + shell: bash + run: | + sleep 1 + sbt buildGraalDistribution + + # Ensure that the versions encoded in the binary and in the release match + - name: Check Versions (Unix) + working-directory: repo + if: runner.os != 'Windows' + shell: bash + run: | + ref=${{ github.ref }} + refversion=${ref#"refs/tags/enso-"} + binversion=${{ env.DIST_VERSION }} + engineversion=$(${{ env.ENGINE_DIST_DIR }}/bin/enso --version --json | jq -r '.version') + test $binversion = $refversion || (echo "Tag version $refversion and the launcher version $binversion do not match" && false) + test $engineversion = $refversion || (echo "Tag version $refversion and the engine version $engineversion do not match" && false) + - name: Check Versions (Windows) + working-directory: repo + if: runner.os == 'Windows' + shell: bash + run: | + ref=${{ github.ref }} + refversion=${ref#"refs/tags/enso-"} + binversion=${{ env.DIST_VERSION }} + engineversion=$(${{ env.ENGINE_DIST_DIR }}/bin/enso.bat --version --json | jq -r '.version') + test $binversion = $refversion || (echo "Tag version $refversion and the launcher version $binversion do not match" && false) + test $engineversion = $refversion || (echo "Tag version $refversion and the engine version $engineversion do not match" && false) + + # Verify License Packages in Distributions + - name: Verify Distributed Licenses Package + working-directory: repo + if: runner.os != 'Windows' # TODO [RW] CRLF handling in licenses task + shell: bash + run: | + sleep 1 + sbt "enso/verifyGeneratedPackage engine ${{ env.ENGINE_DIST_DIR }}/THIRD-PARTY" + sbt "enso/verifyGeneratedPackage launcher ${{ env.LAUNCHER_DIST_DIR }}/THIRD-PARTY" + sbt "enso/verifyGeneratedPackage project-manager ${{ env.PROJECTMANAGER_DIST_DIR }}/THIRD-PARTY" + sbt "enso/verifyGeneratedPackage Standard ${{ env.ENGINE_DIST_DIR }}/std-lib/Standard/THIRD-PARTY" + + # Publish + - name: Upload the Engine Artifact + uses: actions/upload-artifact@v2 + with: + name: ${{ env.ENGINE_DIST_NAME }} + path: repo/${{ env.ENGINE_DIST_ROOT }} + - name: Upload the Launcher Artifact + uses: actions/upload-artifact@v2 + with: + name: ${{ env.LAUNCHER_DIST_NAME }} + path: repo/${{ env.LAUNCHER_DIST_ROOT }} + - name: Upload the Project Manager Artifact + uses: actions/upload-artifact@v2 + with: + name: ${{ env.PROJECTMANAGER_DIST_NAME }} + path: repo/${{ env.PROJECTMANAGER_DIST_ROOT }} + - name: Upload the GraalVM Artifact + uses: actions/upload-artifact@v2 + with: + name: ${{ env.GRAAL_DIST_NAME }} + path: repo/${{ env.GRAAL_DIST_ROOT }} + - name: Upload the Manifest Artifact + uses: actions/upload-artifact@v2 + with: + name: manifest + path: repo/${{ env.ENGINE_DIST_DIR }}/manifest.yaml + - name: Upload the Launcher Manifest Artifact + uses: actions/upload-artifact@v2 + with: + name: launcher-manifest + path: repo/distribution/launcher-manifest.yaml + + create-release: + name: Prepare Release + runs-on: ubuntu-18.04 + needs: build + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + path: repo + # Without specifying options, it downloads all artifacts + - uses: actions/download-artifact@v2 + with: + path: repo/built-distribution + + # This jobs can be used to debug errors, it may be removed + - name: Display Structure of Downloaded Files + run: ls -R + working-directory: repo/built-distribution + + - name: Setup GraalVM Environment + uses: ayltai/setup-graalvm@v1 + with: + graalvm-version: ${{ env.graalVersion }} + java-version: ${{ env.javaVersion }} + native-image: true + - name: Set Up SBT + shell: bash + run: | + curl -fsSL -o sbt.tgz https://github.com/sbt/sbt/releases/download/v${{env.sbtVersion}}/sbt-${{env.sbtVersion}}.tgz + tar -xzf sbt.tgz + echo $GITHUB_WORKSPACE/sbt/bin/ >> $GITHUB_PATH + + # Caches + - name: Cache SBT + uses: actions/cache@v2 + with: + path: | + ~/.sbt + ~/.ivy2/cache + ~/.cache + key: ${{ runner.os }}-sbt-${{ hashFiles('**build.sbt') }} + restore-keys: ${{ runner.os }}-sbt- + + - name: Save Version to Environment + shell: bash + run: | + ref=${{ github.ref }} + DIST_VERSION=${ref#"refs/tags/enso-"} + echo "Preparing release for $DIST_VERSION" + echo "DIST_VERSION=$DIST_VERSION" >> $GITHUB_ENV + + - name: Prepare Packages + shell: bash + working-directory: repo + run: | + sleep 1 + sbt makePackages + + - name: Prepare Bundles + shell: bash + working-directory: repo + run: | + sleep 1 + sbt makeBundles + + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: enso-${{ env.DIST_VERSION }} + release_name: Enso ${{ env.DIST_VERSION }} + body: "Please fill-in the release description" + draft: true + prerelease: true + + - name: Login to ACR + uses: docker/login-action@v1 + with: + registry: ensosharedwus2acr.azurecr.io + username: ${{ secrets.ACR_PUSH_CLIENT_ID }} + password: ${{ secrets.ACR_PUSH_CLIENT_SECRET }} + + - name: Copy the docker entrypoint script + shell: bash + run: | + sleep 1 + cp ./repo/tools/ci/docker/docker-entrypoint.sh ./repo/built-distribution/enso-engine-${{ env.DIST_VERSION }}-linux-amd64/enso-${{ env.DIST_VERSION }}/bin + + - name: Publish Runtime Docker Image + shell: bash + run: | + sleep 1 + docker build -t ensosharedwus2acr.azurecr.io/runtime:${{ env.DIST_VERSION }} -f ./repo/tools/ci/docker/Dockerfile ./repo/built-distribution/enso-engine-${{ env.DIST_VERSION }}-linux-amd64/enso-${{ env.DIST_VERSION }} + docker push ensosharedwus2acr.azurecr.io/runtime:${{ env.DIST_VERSION }} + + # Publish the launcher packages to the backup/fallback S3 bucket + - name: Prepare AWS Session + shell: bash + run: | + aws configure --profile s3-upload <<-EOF > /dev/null 2>&1 + ${{ secrets.LAUNCHER_DEPLOY_ACCESS_KEY_ID }} + ${{ secrets.LAUNCHER_DEPLOY_SECRET_ACCESS_KEY }} + eu-central-1 + text + EOF + - name: Upload the Linux Launcher Package to S3 + shell: bash + run: > + aws s3 cp repo/built-distribution/enso-launcher-${{ env.DIST_VERSION + }}-linux-amd64.tar.gz s3://launcherfallback/launcher/enso-${{ + env.DIST_VERSION }}/ --profile s3-upload --acl public-read + - name: Upload the macOS Launcher Package to S3 + shell: bash + run: > + aws s3 cp repo/built-distribution/enso-launcher-${{ env.DIST_VERSION + }}-macos-amd64.tar.gz s3://launcherfallback/launcher/enso-${{ + env.DIST_VERSION }}/ --profile s3-upload --acl public-read + - name: Upload the Windows Launcher Package to S3 + shell: bash + run: > + aws s3 cp repo/built-distribution/enso-launcher-${{ env.DIST_VERSION + }}-windows-amd64.zip s3://launcherfallback/launcher/enso-${{ + env.DIST_VERSION }}/ --profile s3-upload --acl public-read + - name: Upload the Launcher Manifest to S3 + shell: bash + run: > + aws s3 cp + repo/built-distribution/launcher-manifest/launcher-manifest.yaml + s3://launcherfallback/launcher/enso-${{ env.DIST_VERSION + }}/launcher-manifest.yaml --profile s3-upload --acl public-read + - name: Update the Release List in S3 + shell: bash + run: | + aws s3 cp s3://launcherfallback/release-list.json release-list.json --profile s3-upload + TAG="enso-${{ env.DIST_VERSION }}" + ./repo/tools/ci/releases/add-release.js release-list.json "$TAG" \ + launcher-manifest.yaml \ + "enso-launcher-${{ env.DIST_VERSION }}-linux-amd64.tar.gz" \ + "enso-launcher-${{ env.DIST_VERSION }}-macos-amd64.tar.gz" \ + "enso-launcher-${{ env.DIST_VERSION }}-windows-amd64.zip" + aws s3 cp release-list.json s3://launcherfallback/release-list.json --profile s3-upload --acl public-read + - name: Teardown AWS Session + shell: bash + run: | + aws configure --profile s3-upload <<-EOF > /dev/null 2>&1 + null + null + null + text + EOF + + # Upload the assets to the created release + - name: Publish the Engine (Linux) + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: + repo/built-distribution/enso-engine-${{ env.DIST_VERSION + }}-linux-amd64.tar.gz + asset_name: enso-engine-${{ env.DIST_VERSION }}-linux-amd64.tar.gz + asset_content_type: application/x-tar + - name: Publish the Engine (MacOS) + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: + repo/built-distribution/enso-engine-${{ env.DIST_VERSION + }}-macos-amd64.tar.gz + asset_name: enso-engine-${{ env.DIST_VERSION }}-macos-amd64.tar.gz + asset_content_type: application/x-tar + - name: Publish the Engine (Windows) + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: + repo/built-distribution/enso-engine-${{ env.DIST_VERSION + }}-windows-amd64.zip + asset_name: enso-engine-${{ env.DIST_VERSION }}-windows-amd64.zip + asset_content_type: application/zip + + - name: Publish the Launcher (Linux) + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: + repo/built-distribution/enso-launcher-${{ env.DIST_VERSION + }}-linux-amd64.tar.gz + asset_name: enso-launcher-${{ env.DIST_VERSION }}-linux-amd64.tar.gz + asset_content_type: application/x-tar + - name: Publish the Launcher (MacOS) + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: + repo/built-distribution/enso-launcher-${{ env.DIST_VERSION + }}-macos-amd64.tar.gz + asset_name: enso-launcher-${{ env.DIST_VERSION }}-macos-amd64.tar.gz + asset_content_type: application/x-tar + - name: Publish the Launcher (Windows) + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: + repo/built-distribution/enso-launcher-${{ env.DIST_VERSION + }}-windows-amd64.zip + asset_name: enso-launcher-${{ env.DIST_VERSION }}-windows-amd64.zip + asset_content_type: application/zip + + - name: Publish the Project Manager (Linux) + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: + repo/built-distribution/enso-project-manager-${{ env.DIST_VERSION + }}-linux-amd64.tar.gz + asset_name: + enso-project-manager-${{ env.DIST_VERSION }}-linux-amd64.tar.gz + asset_content_type: application/x-tar + - name: Publish the Project Manager (MacOS) + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: + repo/built-distribution/enso-project-manager-${{ env.DIST_VERSION + }}-macos-amd64.tar.gz + asset_name: + enso-project-manager-${{ env.DIST_VERSION }}-macos-amd64.tar.gz + asset_content_type: application/x-tar + - name: Publish the Project Manager (Windows) + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: + repo/built-distribution/enso-project-manager-${{ env.DIST_VERSION + }}-windows-amd64.zip + asset_name: + enso-project-manager-${{ env.DIST_VERSION }}-windows-amd64.zip + asset_content_type: application/zip + + - name: Publish the Bundle (Linux) + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: + repo/built-distribution/enso-bundle-${{ env.DIST_VERSION + }}-linux-amd64.tar.gz + asset_name: enso-bundle-${{ env.DIST_VERSION }}-linux-amd64.tar.gz + asset_content_type: application/x-tar + - name: Publish the Bundle (MacOS) + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: + repo/built-distribution/enso-bundle-${{ env.DIST_VERSION + }}-macos-amd64.tar.gz + asset_name: enso-bundle-${{ env.DIST_VERSION }}-macos-amd64.tar.gz + asset_content_type: application/x-tar + - name: Publish the Bundle (Windows) + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: + repo/built-distribution/enso-bundle-${{ env.DIST_VERSION + }}-windows-amd64.zip + asset_name: enso-bundle-${{ env.DIST_VERSION }}-windows-amd64.zip + asset_content_type: application/zip + + - name: Publish the Manifest + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: repo/built-distribution/manifest/manifest.yaml + asset_name: manifest.yaml + asset_content_type: application/yaml + - name: Publish the Launcher Manifest + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: repo/built-distribution/launcher-manifest/launcher-manifest.yaml + asset_name: launcher-manifest.yaml + asset_content_type: application/yaml diff --git a/tools/ci/nightly/package.json b/tools/ci/nightly/package.json new file mode 100644 index 000000000000..865e39d39f87 --- /dev/null +++ b/tools/ci/nightly/package.json @@ -0,0 +1,14 @@ +{ + "name": "nightly", + "version": "1.0.0", + "description": "", + "main": "\"\"", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "@octokit/core": "^3.4.0" + } +} diff --git a/tools/ci/nightly/preflight-check.js b/tools/ci/nightly/preflight-check.js new file mode 100644 index 000000000000..c8582c63eb68 --- /dev/null +++ b/tools/ci/nightly/preflight-check.js @@ -0,0 +1,56 @@ +const { Octokit } = require("@octokit/core"); +const octokit = new Octokit(); + +const currentHeadSha = process.argv[2]; +const token = process.env.GITHUB_TOKEN; + +const graphqlWithAuth = octokit.graphql.defaults({ + headers: { + authorization: "token " + token, + }, +}); + +function setProceed(proceed) { + console.log("::set-output name=proceed::" + proceed) +} + +function isNightly(release) { + const nightlyInfix = "-SNAPSHOT-"; + return release.tagName.indexOf(nightlyInfix) >= 0; +} + +async function main() { + const query = ` + { + repository(owner: "enso-org", name: "enso") { + releases(first: 100, orderBy: {direction: DESC, field: CREATED_AT}) { + nodes { + tagName + tagCommit { + oid + } + } + } + } + } + `; + const { repository } = await graphqlWithAuth(query); + const nightlies = repository.releases.nodes.filter(isNightly); + if (nightlies.length == 0) { + console.log("No prior nightly releases found. Proceeding with the first release.") + setProceed(true); + return; + } + + const first = nightlies[0]; + const firstNightlySha = first.tagCommit.oid; + if (firstNightlySha == currentHeadSha) { + console.log("Current commit (" + currentHeadSha + ") is the same as for the most recent nightly build. A new build is not needed.") + setProceed(false); + } else { + console.log("Current commit (" + currentHeadSha + ") is different from the most recent nightly build (" + firstNightlySha + "). Proceeding with a new nightly build.") + setProceed(true); + } +} + +main(); From 8deedfbe95af1486f58bfd169f858627d979e8ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 9 Apr 2021 20:59:40 +0200 Subject: [PATCH 02/30] fix workflow --- .github/workflows/nightly.yml | 24 ++++++++++++------------ tools/ci/nightly/preflight-check.js | 6 +++--- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index fe323519a13d..2d5c9e22ba73 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -25,26 +25,22 @@ jobs: if: > ${{ github.event_name == 'schedule' || contains(github.event.head_commit.message,'[release: nightly]') || true }} outputs: - proceed: - description: Whether the nightly build should proceed. - value: ${{ steps.check-assumptions.outputs.proceed }} - nightly-version: - description: The version string to use for the current build. - value: ${{ steps.compute-version.outputs.nightly-version }} + proceed: ${{ steps.check-assumptions.outputs.proceed }} + nightly-version: ${{ steps.compute-version.outputs.nightly-version }} steps: + - name: Checkout + uses: actions/checkout@v2 - id: check-assumptions name: Check If The Build Should Proceed shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} # The script below sets an output 'proceed' to true or false depending on whether the nightly build should proceed. # Nightly builds are skipped if no new changes are present since the last one. run: | cd tools/ci/nightly/ + npm install node preflight-check.js ${{ github.sha }} - - name: Checkout - if: ${{ steps.check-assumptions.outputs.proceed == 'true' }} - uses: actions/checkout@v2 - with: - path: repo - name: Compute Version id: compute-version if: ${{ steps.check-assumptions.outputs.proceed == 'true' }} @@ -143,6 +139,7 @@ jobs: echo "ENSO_RELEASE_MODE=true" >> $GITHUB_ENV echo "ENSO_VERSION=${{ needs.preflight-check.outputs.nightly-version }}" >> $GITHUB_ENV - name: Update the Version Number to the Nightly + working-directory: repo shell: bash run: | sed -i "s/val ensoVersion.*= \".*\"/val ensoVersion = \"$ENSO_VERSION\"/" build.sbt @@ -361,7 +358,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: tag_name: enso-${{ env.DIST_VERSION }} - release_name: Enso ${{ env.DIST_VERSION }} + release_name: Enso Nightly ${{ env.DIST_VERSION }} body: "Please fill-in the release description" draft: true prerelease: true @@ -600,3 +597,6 @@ jobs: asset_path: repo/built-distribution/launcher-manifest/launcher-manifest.yaml asset_name: launcher-manifest.yaml asset_content_type: application/yaml + + # TODO: publish the release + # Possible hint: secrets.GITHUB_TOKEN may not be allowed to publish releases. diff --git a/tools/ci/nightly/preflight-check.js b/tools/ci/nightly/preflight-check.js index c8582c63eb68..7d2f954e19c1 100644 --- a/tools/ci/nightly/preflight-check.js +++ b/tools/ci/nightly/preflight-check.js @@ -15,8 +15,8 @@ function setProceed(proceed) { } function isNightly(release) { - const nightlyInfix = "-SNAPSHOT-"; - return release.tagName.indexOf(nightlyInfix) >= 0; + const nightlyInfix = "Nightly"; + return release.name.indexOf(nightlyInfix) >= 0; } async function main() { @@ -25,10 +25,10 @@ async function main() { repository(owner: "enso-org", name: "enso") { releases(first: 100, orderBy: {direction: DESC, field: CREATED_AT}) { nodes { - tagName tagCommit { oid } + name } } } From 22c6ac37c39015ffa3d33731af011fec17be561a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Sat, 10 Apr 2021 00:01:40 +0200 Subject: [PATCH 03/30] Refactor, cleaning old releases, auto-publishing --- .github/workflows/nightly.yml | 19 +++++++- tools/ci/nightly/clean-old-releases.js | 31 ++++++++++++ tools/ci/nightly/github.js | 65 ++++++++++++++++++++++++++ tools/ci/nightly/preflight-check.js | 32 +------------ tools/ci/nightly/publish-release.js | 11 +++++ 5 files changed, 126 insertions(+), 32 deletions(-) create mode 100644 tools/ci/nightly/clean-old-releases.js create mode 100644 tools/ci/nightly/github.js create mode 100644 tools/ci/nightly/publish-release.js diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2d5c9e22ba73..0e6b794af8b1 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -598,5 +598,20 @@ jobs: asset_name: launcher-manifest.yaml asset_content_type: application/yaml - # TODO: publish the release - # Possible hint: secrets.GITHUB_TOKEN may not be allowed to publish releases. + - name: Prepare Nodejs + shell: bash + working-directory: repo/tools/ci/nightly + run: npm install + - name: Publish Release + shell: bash + working-directory: repo/tools/ci/nightly + run: node publish-release.js ${{ steps.create_release.id }} + # TODO: secrets.GITHUB_TOKEN may not be allowed to publish releases. + env: + GITHUB_TOKEN: ${{ github.token }} + - name: Remove Old Releases + shell: bash + working-directory: repo/tools/ci/nightly + run: node clean-old-releases.js + env: + GITHUB_TOKEN: ${{ github.token }} diff --git a/tools/ci/nightly/clean-old-releases.js b/tools/ci/nightly/clean-old-releases.js new file mode 100644 index 000000000000..8333c579a8ed --- /dev/null +++ b/tools/ci/nightly/clean-old-releases.js @@ -0,0 +1,31 @@ +const github = require("./github"); + +const MAX_NIGHTLY_RELEASES_TO_KEEP = 3; + +function setProceed(proceed) { + console.log("::set-output name=proceed::" + proceed) +} + +async function main() { + const nightlies = await github.fetchNightlies(); + if (nightlies.length <= MAX_NIGHTLY_RELEASES_TO_KEEP) { + console.log("No releases to clean.") + return; + } + + const toRemoveCount = nightlies.length - MAX_NIGHTLY_RELEASES_TO_KEEP; + if (toRemoveCount > 5) { + console.log("There are " + toRemoveCount + " not-cleaned releases, but every time one is added, older ones are cleaned, so normally there should be at most one to remove. This is very suspicious and it may indicate that the selected set of releases is too broad. Cleaning has been abandoned and the situation should be investigated manually.") + return; + } + + for (let i = MAX_NIGHTLY_RELEASES_TO_KEEP; i < nightlies.length; ++i) { + const id = nightlies[i].id; + console.log("Removing old release " + id + ".") + await github.removeRelease(id); + } + + console.log("Cleanup complete.") +} + +main(); diff --git a/tools/ci/nightly/github.js b/tools/ci/nightly/github.js new file mode 100644 index 000000000000..985cbdb8ff9e --- /dev/null +++ b/tools/ci/nightly/github.js @@ -0,0 +1,65 @@ +const { Octokit } = require("@octokit/core"); +const octokit = new Octokit(); + +const organization = "enso-org"; +const repo = "enso"; +const token = process.env.GITHUB_TOKEN; + +const graphqlWithAuth = octokit.graphql.defaults({ + headers: { + authorization: "token " + token, + }, +}); + +function isNightly(release) { + const nightlyInfix = "Nightly"; + return release.name.indexOf(nightlyInfix) >= 0; +} + +async function fetchAllReleases() { + const query = ` + query getNightlies($owner: String!, $repo: String!) { + repository(owner: $owner, name: $repo) { + releases(first: 100, orderBy: {direction: DESC, field: CREATED_AT}) { + nodes { + tagCommit { + oid + } + name + id + } + } + } + } + `; + const { repository } = await graphqlWithAuth(query, {owner: organization, repo: repo}); + return repository.releases.nodes; +} + +async function fetchNightlies() { + const releases = await fetchAllReleases(); + const nightlies = releases.filter(isNightly); + return nightlies; +} + +async function removeRelease(id) { + await octokit.request('DELETE /repos/{owner}/{repo}/releases/{release_id}', { + owner: organization, + repo: repo, + release_id: id + }); +} + +async function publishRelease(id) { + await octokit.request('PATCH /repos/{owner}/{repo}/releases/{release_id}', { + owner: organization, + repo: repo, + release_id: id, + draft: false + }); +} + +exports.fetchAllReleases = fetchAllReleases; +exports.fetchNightlies = fetchNightlies; +exports.removeRelease = removeRelease; +exports.publishRelease = publishRelease; diff --git a/tools/ci/nightly/preflight-check.js b/tools/ci/nightly/preflight-check.js index 7d2f954e19c1..13ac256226e8 100644 --- a/tools/ci/nightly/preflight-check.js +++ b/tools/ci/nightly/preflight-check.js @@ -1,41 +1,13 @@ -const { Octokit } = require("@octokit/core"); -const octokit = new Octokit(); +const github = require("./github"); const currentHeadSha = process.argv[2]; -const token = process.env.GITHUB_TOKEN; - -const graphqlWithAuth = octokit.graphql.defaults({ - headers: { - authorization: "token " + token, - }, -}); function setProceed(proceed) { console.log("::set-output name=proceed::" + proceed) } -function isNightly(release) { - const nightlyInfix = "Nightly"; - return release.name.indexOf(nightlyInfix) >= 0; -} - async function main() { - const query = ` - { - repository(owner: "enso-org", name: "enso") { - releases(first: 100, orderBy: {direction: DESC, field: CREATED_AT}) { - nodes { - tagCommit { - oid - } - name - } - } - } - } - `; - const { repository } = await graphqlWithAuth(query); - const nightlies = repository.releases.nodes.filter(isNightly); + const nightlies = await github.fetchNightlies(); if (nightlies.length == 0) { console.log("No prior nightly releases found. Proceeding with the first release.") setProceed(true); diff --git a/tools/ci/nightly/publish-release.js b/tools/ci/nightly/publish-release.js new file mode 100644 index 000000000000..f2f40ffdf52f --- /dev/null +++ b/tools/ci/nightly/publish-release.js @@ -0,0 +1,11 @@ +const github = require("./github"); + +const releaseId = process.argv[2]; + +async function main() { + console.log("Making release " + releaseId + " public."); + github.publishRelease(releaseId); + console.log("Done.") +} + +main(); From b9296cff4467cbadb996bdc61ea06b3291767c36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Tue, 13 Apr 2021 18:14:47 +0200 Subject: [PATCH 04/30] Fix version bump script --- .github/workflows/nightly.yml | 6 ++++-- tools/ci/nightly/bump-version.js | 10 ++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 tools/ci/nightly/bump-version.js diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 0e6b794af8b1..18b8c92c836b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -47,7 +47,7 @@ jobs: shell: bash run: | VERSION=$(cat build.sbt | grep 'val ensoVersion' | sed 's/.*"\([^)]*\)".*/\1/').$(date --iso-8601) - echo ::set-output name=proceed::$VERSION + echo ::set-output name=nightly-version::$VERSION # This job should be kept up-to-date with scala.yml#build (but keep the added version check) build: @@ -138,11 +138,13 @@ jobs: run: | echo "ENSO_RELEASE_MODE=true" >> $GITHUB_ENV echo "ENSO_VERSION=${{ needs.preflight-check.outputs.nightly-version }}" >> $GITHUB_ENV + - name: Update the Version Number to the Nightly working-directory: repo shell: bash run: | - sed -i "s/val ensoVersion.*= \".*\"/val ensoVersion = \"$ENSO_VERSION\"/" build.sbt + node tools/ci/nightly/bump-version.js $ENSO_VERSION + - name: Bootstrap the Project working-directory: repo shell: bash diff --git a/tools/ci/nightly/bump-version.js b/tools/ci/nightly/bump-version.js new file mode 100644 index 000000000000..95ecdc2f2bc5 --- /dev/null +++ b/tools/ci/nightly/bump-version.js @@ -0,0 +1,10 @@ +const fs = require('fs'); + +const path = "build.sbt"; +const version = process.argv[2]; + +const content = fs.readFileSync(path, {encoding: "utf-8"}); +const updated = content.replace(/val ensoVersion.*= ".*"/g, 'val ensoVersion = "' + version + '"'); +fs.writeFileSync(path, updated); + +console.log("Updated build version to " + version); From 42a62b500c857ef46b7ffa14d1c04ee98ec86aff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 16 Apr 2021 15:41:44 +0200 Subject: [PATCH 05/30] Fix version check, remove ACR and AWS, fix version in final step --- .github/workflows/nightly.yml | 105 +++++----------------------------- 1 file changed, 13 insertions(+), 92 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 18b8c92c836b..b81061365006 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -30,6 +30,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + - name: Debug Print commit message + shell: bash + run: | + echo ${{ github.event.head_commit.message }} - id: check-assumptions name: Check If The Build Should Proceed shell: bash @@ -49,7 +53,7 @@ jobs: VERSION=$(cat build.sbt | grep 'val ensoVersion' | sed 's/.*"\([^)]*\)".*/\1/').$(date --iso-8601) echo ::set-output name=nightly-version::$VERSION - # This job should be kept up-to-date with scala.yml#build (but keep the added version check) + # This job should be kept up-to-date with scala.yml#build (but keep the relevant changes) build: name: Build runs-on: ${{ matrix.os }} @@ -66,10 +70,6 @@ jobs: uses: actions/checkout@v2 with: path: repo - - name: Debug Print commit message - shell: bash - run: | - echo ${{ github.event.head_commit.message }} - name: Enable Developer Command Prompt (Windows) uses: ilammy/msvc-dev-cmd@v1.5.0 - name: Disable TCP/UDP Offloading (macOS) @@ -226,8 +226,7 @@ jobs: if: runner.os != 'Windows' shell: bash run: | - ref=${{ github.ref }} - refversion=${ref#"refs/tags/enso-"} + refversion=${{ env.ENSO_VERSION }} binversion=${{ env.DIST_VERSION }} engineversion=$(${{ env.ENGINE_DIST_DIR }}/bin/enso --version --json | jq -r '.version') test $binversion = $refversion || (echo "Tag version $refversion and the launcher version $binversion do not match" && false) @@ -237,8 +236,7 @@ jobs: if: runner.os == 'Windows' shell: bash run: | - ref=${{ github.ref }} - refversion=${ref#"refs/tags/enso-"} + refversion=${{ env.ENSO_VERSION }} binversion=${{ env.DIST_VERSION }} engineversion=$(${{ env.ENGINE_DIST_DIR }}/bin/enso.bat --version --json | jq -r '.version') test $binversion = $refversion || (echo "Tag version $refversion and the launcher version $binversion do not match" && false) @@ -291,7 +289,7 @@ jobs: create-release: name: Prepare Release runs-on: ubuntu-18.04 - needs: build + needs: [build, preflight-check] steps: - name: Checkout code uses: actions/checkout@v2 @@ -301,6 +299,10 @@ jobs: - uses: actions/download-artifact@v2 with: path: repo/built-distribution + - name: Prepare Nodejs + shell: bash + working-directory: repo/tools/ci/nightly + run: npm install # This jobs can be used to debug errors, it may be removed - name: Display Structure of Downloaded Files @@ -334,8 +336,7 @@ jobs: - name: Save Version to Environment shell: bash run: | - ref=${{ github.ref }} - DIST_VERSION=${ref#"refs/tags/enso-"} + DIST_VERSION=${{ needs.preflight-check.outputs.nightly-version }} echo "Preparing release for $DIST_VERSION" echo "DIST_VERSION=$DIST_VERSION" >> $GITHUB_ENV @@ -365,82 +366,6 @@ jobs: draft: true prerelease: true - - name: Login to ACR - uses: docker/login-action@v1 - with: - registry: ensosharedwus2acr.azurecr.io - username: ${{ secrets.ACR_PUSH_CLIENT_ID }} - password: ${{ secrets.ACR_PUSH_CLIENT_SECRET }} - - - name: Copy the docker entrypoint script - shell: bash - run: | - sleep 1 - cp ./repo/tools/ci/docker/docker-entrypoint.sh ./repo/built-distribution/enso-engine-${{ env.DIST_VERSION }}-linux-amd64/enso-${{ env.DIST_VERSION }}/bin - - - name: Publish Runtime Docker Image - shell: bash - run: | - sleep 1 - docker build -t ensosharedwus2acr.azurecr.io/runtime:${{ env.DIST_VERSION }} -f ./repo/tools/ci/docker/Dockerfile ./repo/built-distribution/enso-engine-${{ env.DIST_VERSION }}-linux-amd64/enso-${{ env.DIST_VERSION }} - docker push ensosharedwus2acr.azurecr.io/runtime:${{ env.DIST_VERSION }} - - # Publish the launcher packages to the backup/fallback S3 bucket - - name: Prepare AWS Session - shell: bash - run: | - aws configure --profile s3-upload <<-EOF > /dev/null 2>&1 - ${{ secrets.LAUNCHER_DEPLOY_ACCESS_KEY_ID }} - ${{ secrets.LAUNCHER_DEPLOY_SECRET_ACCESS_KEY }} - eu-central-1 - text - EOF - - name: Upload the Linux Launcher Package to S3 - shell: bash - run: > - aws s3 cp repo/built-distribution/enso-launcher-${{ env.DIST_VERSION - }}-linux-amd64.tar.gz s3://launcherfallback/launcher/enso-${{ - env.DIST_VERSION }}/ --profile s3-upload --acl public-read - - name: Upload the macOS Launcher Package to S3 - shell: bash - run: > - aws s3 cp repo/built-distribution/enso-launcher-${{ env.DIST_VERSION - }}-macos-amd64.tar.gz s3://launcherfallback/launcher/enso-${{ - env.DIST_VERSION }}/ --profile s3-upload --acl public-read - - name: Upload the Windows Launcher Package to S3 - shell: bash - run: > - aws s3 cp repo/built-distribution/enso-launcher-${{ env.DIST_VERSION - }}-windows-amd64.zip s3://launcherfallback/launcher/enso-${{ - env.DIST_VERSION }}/ --profile s3-upload --acl public-read - - name: Upload the Launcher Manifest to S3 - shell: bash - run: > - aws s3 cp - repo/built-distribution/launcher-manifest/launcher-manifest.yaml - s3://launcherfallback/launcher/enso-${{ env.DIST_VERSION - }}/launcher-manifest.yaml --profile s3-upload --acl public-read - - name: Update the Release List in S3 - shell: bash - run: | - aws s3 cp s3://launcherfallback/release-list.json release-list.json --profile s3-upload - TAG="enso-${{ env.DIST_VERSION }}" - ./repo/tools/ci/releases/add-release.js release-list.json "$TAG" \ - launcher-manifest.yaml \ - "enso-launcher-${{ env.DIST_VERSION }}-linux-amd64.tar.gz" \ - "enso-launcher-${{ env.DIST_VERSION }}-macos-amd64.tar.gz" \ - "enso-launcher-${{ env.DIST_VERSION }}-windows-amd64.zip" - aws s3 cp release-list.json s3://launcherfallback/release-list.json --profile s3-upload --acl public-read - - name: Teardown AWS Session - shell: bash - run: | - aws configure --profile s3-upload <<-EOF > /dev/null 2>&1 - null - null - null - text - EOF - # Upload the assets to the created release - name: Publish the Engine (Linux) uses: actions/upload-release-asset@v1 @@ -600,10 +525,6 @@ jobs: asset_name: launcher-manifest.yaml asset_content_type: application/yaml - - name: Prepare Nodejs - shell: bash - working-directory: repo/tools/ci/nightly - run: npm install - name: Publish Release shell: bash working-directory: repo/tools/ci/nightly From 8f975b82c9c4eed153c137ef498e3d294a0540e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 16 Apr 2021 16:23:41 +0200 Subject: [PATCH 06/30] Fix version in last step, add release notes --- .github/workflows/nightly.yml | 15 +++++++-- .github/workflows/release.yml | 5 --- RELEASES.md | 6 ++++ tools/ci/nightly/extract-release-notes.js | 40 +++++++++++++++++++++++ 4 files changed, 59 insertions(+), 7 deletions(-) create mode 100644 tools/ci/nightly/extract-release-notes.js diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b81061365006..f3a40d4c8c2d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -306,7 +306,7 @@ jobs: # This jobs can be used to debug errors, it may be removed - name: Display Structure of Downloaded Files - run: ls -R + run: ls working-directory: repo/built-distribution - name: Setup GraalVM Environment @@ -339,6 +339,11 @@ jobs: DIST_VERSION=${{ needs.preflight-check.outputs.nightly-version }} echo "Preparing release for $DIST_VERSION" echo "DIST_VERSION=$DIST_VERSION" >> $GITHUB_ENV + - name: Update the Version Number to the Nightly + working-directory: repo + shell: bash + run: | + node tools/ci/nightly/bump-version.js $DIST_VERSION - name: Prepare Packages shell: bash @@ -354,6 +359,12 @@ jobs: sleep 1 sbt makeBundles + - name: Prepare Release Notes + working-directory: repo + shell: bash + run: | + node tools/ci/nightly/extract-release-notes.js RELEASES.md release_notes.md + - name: Create Release id: create_release uses: actions/create-release@v1 @@ -362,7 +373,7 @@ jobs: with: tag_name: enso-${{ env.DIST_VERSION }} release_name: Enso Nightly ${{ env.DIST_VERSION }} - body: "Please fill-in the release description" + body_path: release_notes.md draft: true prerelease: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cbe5052cb450..2d50e353d10d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -253,11 +253,6 @@ jobs: with: path: repo/built-distribution - # This jobs can be used to debug errors, it may be removed - - name: Display Structure of Downloaded Files - run: ls -R - working-directory: repo/built-distribution - - name: Setup GraalVM Environment uses: ayltai/setup-graalvm@v1 with: diff --git a/RELEASES.md b/RELEASES.md index 18abd6f9df80..4153be6890f8 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,3 +1,9 @@ +# Nightly + +## Miscellaneous + +- Adding a pipeline for automatic nightly builds. + # Enso 0.2.11 (2021-04-28) ## Tooling diff --git a/tools/ci/nightly/extract-release-notes.js b/tools/ci/nightly/extract-release-notes.js new file mode 100644 index 000000000000..a1d957037722 --- /dev/null +++ b/tools/ci/nightly/extract-release-notes.js @@ -0,0 +1,40 @@ +const fs = require('fs'); + +const inputPath = process.argv[2]; +const outputPath = process.argv[3]; + +console.log("Extracting release notes from " + inputPath + " to " + outputPath); + +/** Returns the part of the text until the second top-level heading (exclusive) + * in Markdown formatting. + */ +function cutFirstSection(content) { + // TODO [RW] can we assume that '# ' is always the start of a section? + // This requires that there are no code snippets with comments starting at the + // beginning of the line. + function findSectionStart(text) { + return text.search(/^# /gm); + } + + const firstHeading = findSectionStart(content); + if (firstHeading < 0) { + throw "No sections in file!"; + } + const restStart = firstHeading + 2; + const rest = content.substring(restStart); + const secondHeading = findSectionStart(rest); + if (secondHeading < 0) { + // There is only one section. + return content; + } + const secondHeadingOffsetInContent = restStart + secondHeading; + const firstSectionContent = content.substring(0, secondHeadingOffsetInContent); + return firstSectionContent; +} + +const content = fs.readFileSync(inputPath, {encoding: "utf-8"}); +const nightlyPart = cutFirstSection(content); +fs.writeFileSync(outputPath, nightlyPart); + +console.log("Created " + outputPath + " with the following content:"); +console.log(nightlyPart); From bd4fa94a73095cbd7993e0a9d2e28af3d1bcab38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 16 Apr 2021 17:50:47 +0200 Subject: [PATCH 07/30] Add a CHANGELOG check For now, not skippable --- .github/workflows/docs.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d24da1319af3..42cfb35be7a8 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -27,3 +27,22 @@ jobs: run: npm install - name: Check Docs run: npx prettier --check . + changelog-check: + name: Changelog Check + runs-on: ubuntu-latest + timeout-minutes: 10 + if: ${{ github.event_name == 'pull_request' }} + steps: + - name: Checkout Sources + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Install Node + uses: actions/setup-node@v1 + with: + node-version: ${{ env.nodeVersion }} + # TODO how to indicate that we don't want to check? Is it enough that any commit of the PR has `no-changelog` ? + - name: Check If Changelog Was Modified + shell: bash + run: | + ! git --no-pager diff --exit-code ${{ }} RELEASES.md From f055daeebb128f6439388ac4ee18b2fd9fc26d78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 16 Apr 2021 17:52:49 +0200 Subject: [PATCH 08/30] Fix path --- .github/workflows/docs.yml | 3 +-- .github/workflows/nightly.yml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 42cfb35be7a8..b716f64750eb 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -44,5 +44,4 @@ jobs: # TODO how to indicate that we don't want to check? Is it enough that any commit of the PR has `no-changelog` ? - name: Check If Changelog Was Modified shell: bash - run: | - ! git --no-pager diff --exit-code ${{ }} RELEASES.md + run: "! git --no-pager diff --exit-code ${{ }} RELEASES.md" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index f3a40d4c8c2d..358a1075b9fb 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -373,7 +373,7 @@ jobs: with: tag_name: enso-${{ env.DIST_VERSION }} release_name: Enso Nightly ${{ env.DIST_VERSION }} - body_path: release_notes.md + body_path: repo/release_notes.md draft: true prerelease: true From 13718a67013bd59378ac01a194bf31a8af3b4ae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 16 Apr 2021 20:20:21 +0200 Subject: [PATCH 09/30] Fixes --- .github/workflows/docs.yml | 3 +- .github/workflows/nightly.yml | 2 +- tools/ci/check-changelog.js | 17 +++++++++ tools/ci/nightly/clean-old-releases.js | 5 ++- tools/ci/nightly/github.js | 51 +++++++++++++------------- tools/ci/nightly/preflight-check.js | 5 ++- tools/ci/nightly/publish-release.js | 7 +++- 7 files changed, 58 insertions(+), 32 deletions(-) create mode 100644 tools/ci/check-changelog.js diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index b716f64750eb..7bc73b5ca6e4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -44,4 +44,5 @@ jobs: # TODO how to indicate that we don't want to check? Is it enough that any commit of the PR has `no-changelog` ? - name: Check If Changelog Was Modified shell: bash - run: "! git --no-pager diff --exit-code ${{ }} RELEASES.md" + run: | + node tools/ci/check-changelog.js RELEASES.md ${{ github.base_ref }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 358a1075b9fb..6b028ed85362 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -539,7 +539,7 @@ jobs: - name: Publish Release shell: bash working-directory: repo/tools/ci/nightly - run: node publish-release.js ${{ steps.create_release.id }} + run: node publish-release.js ${{ steps.create_release.outputs.id }} # TODO: secrets.GITHUB_TOKEN may not be allowed to publish releases. env: GITHUB_TOKEN: ${{ github.token }} diff --git a/tools/ci/check-changelog.js b/tools/ci/check-changelog.js new file mode 100644 index 000000000000..cfb2b6b4cc93 --- /dev/null +++ b/tools/ci/check-changelog.js @@ -0,0 +1,17 @@ +const fs = require('fs'); +const proc = require('child_process'); + +const changelogPath = process.argv[2]; +const ref = process.argv[3]; + +const diffArgs = ["--no-pager", "diff", "--exit-code", ref, changelogPath]; +const exitCode = proc.spawnSync("git", diffArgs).status; + +if (exitCode == 0) { + console.log("No changes to the changelog"); + process.exit(1); +} else { + console.log("Changelog was changed"); + process.exit(0); +} + diff --git a/tools/ci/nightly/clean-old-releases.js b/tools/ci/nightly/clean-old-releases.js index 8333c579a8ed..93996cb6d055 100644 --- a/tools/ci/nightly/clean-old-releases.js +++ b/tools/ci/nightly/clean-old-releases.js @@ -28,4 +28,7 @@ async function main() { console.log("Cleanup complete.") } -main(); +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/tools/ci/nightly/github.js b/tools/ci/nightly/github.js index 985cbdb8ff9e..cdab8e0ce9ad 100644 --- a/tools/ci/nightly/github.js +++ b/tools/ci/nightly/github.js @@ -2,14 +2,25 @@ const { Octokit } = require("@octokit/core"); const octokit = new Octokit(); const organization = "enso-org"; -const repo = "enso"; -const token = process.env.GITHUB_TOKEN; +function determineRepositoryName() { + const fallback = "enso"; + const fallbackMessage = "Could not determine the repository name, falling back to the default."; + const fullName = process.env.GITHUB_REPOSITORY; + if (!fullName) { + console.log(fallbackMessage); + return fallback; + } -const graphqlWithAuth = octokit.graphql.defaults({ - headers: { - authorization: "token " + token, - }, -}); + const prefix = organization + "/"; + if (fullName.startsWith(prefix)) { + return fullName.substring(prefix.length); + } else { + console.log(fallbackMessage); + return fallback; + } +} +const repo = determineRepositoryName(); +const token = process.env.GITHUB_TOKEN; function isNightly(release) { const nightlyInfix = "Nightly"; @@ -17,23 +28,11 @@ function isNightly(release) { } async function fetchAllReleases() { - const query = ` - query getNightlies($owner: String!, $repo: String!) { - repository(owner: $owner, name: $repo) { - releases(first: 100, orderBy: {direction: DESC, field: CREATED_AT}) { - nodes { - tagCommit { - oid - } - name - id - } - } - } - } - `; - const { repository } = await graphqlWithAuth(query, {owner: organization, repo: repo}); - return repository.releases.nodes; + const res = await octokit.request('GET /repos/{owner}/{repo}/releases', { + owner: organization, + repo: repo + }); + return res.data; } async function fetchNightlies() { @@ -43,7 +42,7 @@ async function fetchNightlies() { } async function removeRelease(id) { - await octokit.request('DELETE /repos/{owner}/{repo}/releases/{release_id}', { + return await octokit.request('DELETE /repos/{owner}/{repo}/releases/{release_id}', { owner: organization, repo: repo, release_id: id @@ -51,7 +50,7 @@ async function removeRelease(id) { } async function publishRelease(id) { - await octokit.request('PATCH /repos/{owner}/{repo}/releases/{release_id}', { + return await octokit.request('PATCH /repos/{owner}/{repo}/releases/{release_id}', { owner: organization, repo: repo, release_id: id, diff --git a/tools/ci/nightly/preflight-check.js b/tools/ci/nightly/preflight-check.js index 13ac256226e8..d31934ba0c0e 100644 --- a/tools/ci/nightly/preflight-check.js +++ b/tools/ci/nightly/preflight-check.js @@ -25,4 +25,7 @@ async function main() { } } -main(); +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/tools/ci/nightly/publish-release.js b/tools/ci/nightly/publish-release.js index f2f40ffdf52f..fdcb2b35dc9d 100644 --- a/tools/ci/nightly/publish-release.js +++ b/tools/ci/nightly/publish-release.js @@ -4,8 +4,11 @@ const releaseId = process.argv[2]; async function main() { console.log("Making release " + releaseId + " public."); - github.publishRelease(releaseId); + await github.publishRelease(releaseId); console.log("Done.") } -main(); +main().catch(err => { + console.error(err); + process.exit(1); +}); From 7a97ea622f7ecc92ec9e00e8dcb6778aaea5bffc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Tue, 20 Apr 2021 18:07:57 +0200 Subject: [PATCH 10/30] WIP workflow trigger by webhook --- .github/workflows/nightly.yml | 8 ++++++++ tools/ci/nightly/github.js | 10 ++++++++++ tools/ci/nightly/trigger-workflow.js | 16 ++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 tools/ci/nightly/trigger-workflow.js diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6b028ed85362..e19a4e76766a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -536,6 +536,7 @@ jobs: asset_name: launcher-manifest.yaml asset_content_type: application/yaml + # TODO what to do about this issue - name: Publish Release shell: bash working-directory: repo/tools/ci/nightly @@ -543,6 +544,13 @@ jobs: # TODO: secrets.GITHUB_TOKEN may not be allowed to publish releases. env: GITHUB_TOKEN: ${{ github.token }} + # TODO enable this once IDE supports triggers +# - name: Trigger IDE Build +# shell: bash +# working-directory: repo/tools/ci/nightly +# run: node trigger-workflow.js ide gui-ci develop +# env: +# GITHUB_TOKEN: ${{ github.token }} - name: Remove Old Releases shell: bash working-directory: repo/tools/ci/nightly diff --git a/tools/ci/nightly/github.js b/tools/ci/nightly/github.js index cdab8e0ce9ad..53049597b9f3 100644 --- a/tools/ci/nightly/github.js +++ b/tools/ci/nightly/github.js @@ -58,7 +58,17 @@ async function publishRelease(id) { }); } +async function triggerWorkflow(repo, workflow_id, ref) { + await octokit.request('POST /repos/{owner}/{repo}/actions/workflows/{workflow_id}/dispatches', { + owner: organization, + repo: repo, + workflow_id: workflow_id, + ref: ref + }) +} + exports.fetchAllReleases = fetchAllReleases; exports.fetchNightlies = fetchNightlies; exports.removeRelease = removeRelease; exports.publishRelease = publishRelease; +exports.repository = repo; diff --git a/tools/ci/nightly/trigger-workflow.js b/tools/ci/nightly/trigger-workflow.js new file mode 100644 index 000000000000..cb74e4fbe1a5 --- /dev/null +++ b/tools/ci/nightly/trigger-workflow.js @@ -0,0 +1,16 @@ +const github = require("./github"); + +const repo = process.argv[2]; +const workflow_id = process.argv[3]; +const ref = process.argv[4]; + +async function main() { + console.log("Triggering workflow " + workflow_id + " in " + repo + " on " + ref); + await github.triggerWorkflow(repo, workflow_id, ref); + console.log("Done.") +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); From 5973b82641e600ca980a76677e49d91a00a57a35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Tue, 20 Apr 2021 18:12:55 +0200 Subject: [PATCH 11/30] Assume that we workaround the publishing issue by publishing before upload --- .github/workflows/nightly.yml | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e19a4e76766a..f88b60918cb3 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -4,7 +4,7 @@ name: Nightly Release CI on: push: schedule: - - cron: '0 4 * * 2-6' # 4am from Tuesday to Saturday (i.e. after every workday) + - cron: "0 4 * * 2-6" # 4am from Tuesday to Saturday (i.e. after every workday) env: # Please ensure that this is in sync with graalVersion in build.sbt @@ -23,7 +23,8 @@ jobs: timeout-minutes: 10 # That true here is for debugging on staging if: > - ${{ github.event_name == 'schedule' || contains(github.event.head_commit.message,'[release: nightly]') || true }} + ${{ github.event_name == 'schedule' || + contains(github.event.head_commit.message,'[release: nightly]') || true }} outputs: proceed: ${{ steps.check-assumptions.outputs.proceed }} nightly-version: ${{ steps.compute-version.outputs.nightly-version }} @@ -374,7 +375,7 @@ jobs: tag_name: enso-${{ env.DIST_VERSION }} release_name: Enso Nightly ${{ env.DIST_VERSION }} body_path: repo/release_notes.md - draft: true + draft: false prerelease: true # Upload the assets to the created release @@ -536,21 +537,21 @@ jobs: asset_name: launcher-manifest.yaml asset_content_type: application/yaml - # TODO what to do about this issue - - name: Publish Release - shell: bash - working-directory: repo/tools/ci/nightly - run: node publish-release.js ${{ steps.create_release.outputs.id }} - # TODO: secrets.GITHUB_TOKEN may not be allowed to publish releases. - env: - GITHUB_TOKEN: ${{ github.token }} + # # TODO what to do about this issue, temporarily I just publish immediately + # - name: Publish Release + # shell: bash + # working-directory: repo/tools/ci/nightly + # run: node publish-release.js ${{ steps.create_release.outputs.id }} + # # TODO: secrets.GITHUB_TOKEN may not be allowed to publish releases. + # env: + # GITHUB_TOKEN: ${{ github.token }} # TODO enable this once IDE supports triggers -# - name: Trigger IDE Build -# shell: bash -# working-directory: repo/tools/ci/nightly -# run: node trigger-workflow.js ide gui-ci develop -# env: -# GITHUB_TOKEN: ${{ github.token }} + # - name: Trigger IDE Build + # shell: bash + # working-directory: repo/tools/ci/nightly + # run: node trigger-workflow.js ide gui-ci develop + # env: + # GITHUB_TOKEN: ${{ github.token }} - name: Remove Old Releases shell: bash working-directory: repo/tools/ci/nightly From 8d01f89d43806803426aa5755503cf5302e22720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Tue, 20 Apr 2021 22:47:32 +0200 Subject: [PATCH 12/30] WIP adding warnings regarding nightlies --- .../components/Engine.scala | 22 ++++++++++++++ .../releases/ReleaseProviderException.scala | 29 +++++++++++++++---- .../engine/EngineReleaseProvider.scala | 23 +++++++++++++-- .../releases/github/GithubAPI.scala | 8 +++-- .../runtimeversionmanager/runner/Runner.scala | 9 +++++- 5 files changed, 79 insertions(+), 12 deletions(-) diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Engine.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Engine.scala index d777487201a4..830f28716296 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Engine.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Engine.scala @@ -67,4 +67,26 @@ case class Engine(version: SemVer, path: Path, manifest: Manifest) { * installed release and issue a warning when they are used. */ def isMarkedBroken: Boolean = manifest.brokenMark + + /** Specifies if the engine version comes from a nightly release. + * + * See `docs/distribution/nightly.md` for more information. TODO [RW] docs + */ + def isNightly: Boolean = Engine.isNightly(version) +} + +object Engine { + + /** The infix that should be included in the pre-release part of the semantic + * versioning string that describes the engine version to indicate that this + * is a prerelease. + */ + def nightlyInfix = "SNAPSHOT" + + /** Specifies if the engine version comes from a nightly release. + * + * See `docs/distribution/nightly.md` for more information. + */ + def isNightly(version: SemVer): Boolean = + version.preRelease.exists(_.contains(nightlyInfix)) } diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/ReleaseProviderException.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/ReleaseProviderException.scala index 0e4b883931e2..7d742c5b8dc1 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/ReleaseProviderException.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/ReleaseProviderException.scala @@ -1,12 +1,31 @@ package org.enso.runtimeversionmanager.releases -/** Indicates a release provider failure. - */ -case class ReleaseProviderException(message: String, cause: Throwable = null) +/** Indicates a release provider failure. */ +sealed class ReleaseProviderException(message: String, cause: Throwable) extends RuntimeException(message, cause) { - /** @inheritdoc - */ + /** @inheritdoc */ override def toString: String = s"A problem occurred when trying to find the release: $message" } + +object ReleaseProviderException { + + /** Creates a release provider exception with a given message and optional + * cause. + */ + def apply( + message: String, + cause: Throwable = null + ): ReleaseProviderException = new ReleaseProviderException(message, cause) +} + +/** Indicates that the specific release could not be found. + * + * @param tag the tag of the release + * @param cause (optional) an exception that has caused this error + */ +case class ReleaseNotFound( + tag: String, + cause: Throwable = null +) extends ReleaseProviderException(s"Cannot find release `$tag`.", cause) diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/engine/EngineReleaseProvider.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/engine/EngineReleaseProvider.scala index 35dd533500aa..373a585556e5 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/engine/EngineReleaseProvider.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/engine/EngineReleaseProvider.scala @@ -1,13 +1,13 @@ package org.enso.runtimeversionmanager.releases.engine import java.nio.file.Path - import nl.gn0s1s.bump.SemVer import org.enso.cli.task.TaskProgress -import org.enso.runtimeversionmanager.components.Manifest +import org.enso.runtimeversionmanager.components.{Engine, Manifest} import org.enso.runtimeversionmanager.releases.{ EnsoReleaseProvider, Release, + ReleaseNotFound, ReleaseProviderException, SimpleReleaseProvider } @@ -22,7 +22,7 @@ class EngineReleaseProvider(releaseProvider: SimpleReleaseProvider) def fetchRelease(version: SemVer): Try[EngineRelease] = { val tag = tagPrefix + version.toString for { - release <- releaseProvider.releaseForTag(tag) + release <- wrapFetchError(releaseProvider.releaseForTag(tag)) manifestAsset <- release.assets .find(_.fileName == Manifest.DEFAULT_MANIFEST_NAME) @@ -53,6 +53,23 @@ class EngineReleaseProvider(releaseProvider: SimpleReleaseProvider) ) } + /** Intercepts the `ReleaseNotFound` and injects a message regarding nightly + * releases if applicable. + */ + private def wrapFetchError[A](result: Try[A]): Try[A] = result.recoverWith { + case ReleaseNotFound(tag, cause) if tag.contains(Engine.nightlyInfix) => + // TODO [RW] shouldn't we somehow explain *how* to upgrade? Ideally the + // launcher should include a command for that. + Failure( + ReleaseNotFound( + s"Cannot find release `$tag`. Nightly releases expire after some " + + s"time. Consider upgrading to a stable release or a newer nightly " + + s"build.", + cause + ) + ) + } + private case class DefaultEngineRelease( version: SemVer, manifest: Manifest, diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/github/GithubAPI.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/github/GithubAPI.scala index 9d1b8eb74eda..c8cbdca83266 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/github/GithubAPI.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/github/GithubAPI.scala @@ -1,7 +1,6 @@ package org.enso.runtimeversionmanager.releases.github import java.nio.file.Path - import io.circe._ import io.circe.parser._ import org.enso.cli.task.TaskProgress @@ -12,7 +11,10 @@ import org.enso.runtimeversionmanager.http.{ Header, URIBuilder } -import org.enso.runtimeversionmanager.releases.ReleaseProviderException +import org.enso.runtimeversionmanager.releases.{ + ReleaseNotFound, + ReleaseProviderException +} import scala.util.{Success, Try} @@ -107,7 +109,7 @@ object GithubAPI { .map(err => handleError( response, - ReleaseProviderException(s"Cannot find release `$tag`.", err) + ReleaseNotFound(tag, err) ) ) .toTry diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/runner/Runner.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/runner/Runner.scala index cb9c7fdc5dfc..d00292aea905 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/runner/Runner.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/runner/Runner.scala @@ -1,7 +1,6 @@ package org.enso.runtimeversionmanager.runner import java.nio.file.Path - import akka.http.scaladsl.model.Uri import com.typesafe.scalalogging.Logger import nl.gn0s1s.bump.SemVer @@ -55,6 +54,14 @@ class Runner( "--new-project-name", name ) ++ authorNameOption ++ authorEmailOption ++ additionalArguments + // TODO [RW] this warning will not be clearly visible in the IDE, do we want some other mechanism there? It would most likely require modifying the protocol. + if (Engine.isNightly(engineVersion)) { + Logger[Runner].warn( + "Creating a new project using a nightly build. Nightly builds may " + + "disappear after a while, so you may need to upgrade. Consider " + + "using a stable version." + ) + } RunSettings(engineVersion, arguments, connectLoggerIfAvailable = false) } From f0c15832fc98cfd769bf64096432990bf9e841a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Tue, 20 Apr 2021 23:09:31 +0200 Subject: [PATCH 13/30] Avoid crashing on a Promise race. TODO: ideally it should be investigated why this race happens for programs running for a very short time --- .../LoggingServiceSetupHelper.scala | 36 ++++++++++++++----- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/lib/scala/logging-service/src/main/scala/org/enso/loggingservice/LoggingServiceSetupHelper.scala b/lib/scala/logging-service/src/main/scala/org/enso/loggingservice/LoggingServiceSetupHelper.scala index 738845bf2457..564fcb1ce348 100644 --- a/lib/scala/logging-service/src/main/scala/org/enso/loggingservice/LoggingServiceSetupHelper.scala +++ b/lib/scala/logging-service/src/main/scala/org/enso/loggingservice/LoggingServiceSetupHelper.scala @@ -180,10 +180,20 @@ abstract class LoggingServiceSetupHelper(implicit } case Success(serverBinding) => val uri = serverBinding.toUri() - loggingServiceEndpointPromise.success(Some(uri)) - logger.trace( - s"Logging service has been set-up and is listening at `$uri`." - ) + try { + loggingServiceEndpointPromise.success(Some(uri)) + logger.trace( + s"Logging service has been set-up and is listening at `$uri`." + ) + } catch { + case _: IllegalStateException => + val earlierValue = loggingServiceEndpointPromise.future.value + logger.warn( + s"The logging service has been set-up at `$uri`, but the " + + s"logging URI has been initialized before that to " + + s"$earlierValue." + ) + } } } @@ -214,10 +224,20 @@ abstract class LoggingServiceSetupHelper(implicit loggingServiceEndpointPromise.trySuccess(None) case Success(connected) => if (connected) { - loggingServiceEndpointPromise.success(Some(uri)) - System.err.println( - s"Log messages are forwarded to `$uri`." - ) + try { + loggingServiceEndpointPromise.success(Some(uri)) + System.err.println( + s"Log messages are forwarded to `$uri`." + ) + } catch { + case _: IllegalStateException => + val earlierValue = loggingServiceEndpointPromise.future.value + logger.warn( + s"The logging service has been set-up at `$uri`, but the " + + s"logging URI has been initialized before that to " + + s"$earlierValue." + ) + } } else { loggingServiceEndpointPromise.trySuccess(None) } From fa723d86d78a3979165faaba0056a239d722bf23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Apr 2021 13:08:47 +0200 Subject: [PATCH 14/30] Fix preflight check, remove deprecated publish helper --- .github/workflows/nightly.yml | 8 -------- tools/ci/nightly/github.js | 12 +----------- tools/ci/nightly/preflight-check.js | 2 +- tools/ci/nightly/publish-release.js | 14 -------------- 4 files changed, 2 insertions(+), 34 deletions(-) delete mode 100644 tools/ci/nightly/publish-release.js diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index f88b60918cb3..3d2691878a74 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -537,14 +537,6 @@ jobs: asset_name: launcher-manifest.yaml asset_content_type: application/yaml - # # TODO what to do about this issue, temporarily I just publish immediately - # - name: Publish Release - # shell: bash - # working-directory: repo/tools/ci/nightly - # run: node publish-release.js ${{ steps.create_release.outputs.id }} - # # TODO: secrets.GITHUB_TOKEN may not be allowed to publish releases. - # env: - # GITHUB_TOKEN: ${{ github.token }} # TODO enable this once IDE supports triggers # - name: Trigger IDE Build # shell: bash diff --git a/tools/ci/nightly/github.js b/tools/ci/nightly/github.js index 53049597b9f3..e8c81f3bd055 100644 --- a/tools/ci/nightly/github.js +++ b/tools/ci/nightly/github.js @@ -24,7 +24,7 @@ const token = process.env.GITHUB_TOKEN; function isNightly(release) { const nightlyInfix = "Nightly"; - return release.name.indexOf(nightlyInfix) >= 0; + return release.name.indexOf(nightlyInfix) >= 0 && !release.draft; } async function fetchAllReleases() { @@ -49,15 +49,6 @@ async function removeRelease(id) { }); } -async function publishRelease(id) { - return await octokit.request('PATCH /repos/{owner}/{repo}/releases/{release_id}', { - owner: organization, - repo: repo, - release_id: id, - draft: false - }); -} - async function triggerWorkflow(repo, workflow_id, ref) { await octokit.request('POST /repos/{owner}/{repo}/actions/workflows/{workflow_id}/dispatches', { owner: organization, @@ -70,5 +61,4 @@ async function triggerWorkflow(repo, workflow_id, ref) { exports.fetchAllReleases = fetchAllReleases; exports.fetchNightlies = fetchNightlies; exports.removeRelease = removeRelease; -exports.publishRelease = publishRelease; exports.repository = repo; diff --git a/tools/ci/nightly/preflight-check.js b/tools/ci/nightly/preflight-check.js index d31934ba0c0e..771cc31a8b1b 100644 --- a/tools/ci/nightly/preflight-check.js +++ b/tools/ci/nightly/preflight-check.js @@ -15,7 +15,7 @@ async function main() { } const first = nightlies[0]; - const firstNightlySha = first.tagCommit.oid; + const firstNightlySha = first.target_commitish; if (firstNightlySha == currentHeadSha) { console.log("Current commit (" + currentHeadSha + ") is the same as for the most recent nightly build. A new build is not needed.") setProceed(false); diff --git a/tools/ci/nightly/publish-release.js b/tools/ci/nightly/publish-release.js deleted file mode 100644 index fdcb2b35dc9d..000000000000 --- a/tools/ci/nightly/publish-release.js +++ /dev/null @@ -1,14 +0,0 @@ -const github = require("./github"); - -const releaseId = process.argv[2]; - -async function main() { - console.log("Making release " + releaseId + " public."); - await github.publishRelease(releaseId); - console.log("Done.") -} - -main().catch(err => { - console.error(err); - process.exit(1); -}); From 1f7c3e65789b7feba9fa1e13cb95acc93efed8ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Apr 2021 13:40:49 +0200 Subject: [PATCH 15/30] Check changelog --- tools/ci/check-changelog.js | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/tools/ci/check-changelog.js b/tools/ci/check-changelog.js index cfb2b6b4cc93..4e364ab82c6c 100644 --- a/tools/ci/check-changelog.js +++ b/tools/ci/check-changelog.js @@ -1,17 +1,33 @@ const fs = require('fs'); const proc = require('child_process'); +const skipChangelogInfix = "[no-changelog]" const changelogPath = process.argv[2]; -const ref = process.argv[3]; +const baseRef = process.argv[3]; -const diffArgs = ["--no-pager", "diff", "--exit-code", ref, changelogPath]; -const exitCode = proc.spawnSync("git", diffArgs).status; +function wasChangelogModified() { + const diffArgs = ["--no-pager", "diff", "--exit-code", ref, changelogPath]; + const exitCode = proc.spawnSync("git", diffArgs).status; + const noDifference = exitCode == 0; + return !noDifference; +} -if (exitCode == 0) { - console.log("No changes to the changelog"); - process.exit(1); -} else { - console.log("Changelog was changed"); - process.exit(0); +function isChangelogSkipped() { + const logArgs = ["--no-pager", "log", "HEAD~3...HEAD", "--pretty=oneline"] + const output = proc.spawnSync("git", diffArgs).stdout.toString("utf-8"); + const containsSkipCommit = output.indexOf(skipChangelogInfix) >= 0; + return containsSkipCommit; } +if (wasChangelogModified()) { + console.log("Changelog was changed"); + process.exit(0); +} else { + console.log("No changes to the changelog"); + if (isChangelogSkipped()) { + console.log("But one of the commits within the PR includes " + skipChangelogInfix + ", so the check is skipped."); + process.exit(0); + } else { + process.exit(1); + } +} From 0a0c79fc815cc576c73538a20f9f0e719cf1ae77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Apr 2021 13:41:30 +0200 Subject: [PATCH 16/30] JS formatting --- tools/ci/nightly/bump-version.js | 9 ++++-- tools/ci/nightly/clean-old-releases.js | 16 +++++++---- tools/ci/nightly/extract-release-notes.js | 13 +++++---- tools/ci/nightly/github.js | 35 ++++++++++++++--------- tools/ci/nightly/preflight-check.js | 22 ++++++++++---- tools/ci/nightly/trigger-workflow.js | 8 ++++-- 6 files changed, 67 insertions(+), 36 deletions(-) diff --git a/tools/ci/nightly/bump-version.js b/tools/ci/nightly/bump-version.js index 95ecdc2f2bc5..6636dfaf2967 100644 --- a/tools/ci/nightly/bump-version.js +++ b/tools/ci/nightly/bump-version.js @@ -1,10 +1,13 @@ -const fs = require('fs'); +const fs = require("fs"); const path = "build.sbt"; const version = process.argv[2]; -const content = fs.readFileSync(path, {encoding: "utf-8"}); -const updated = content.replace(/val ensoVersion.*= ".*"/g, 'val ensoVersion = "' + version + '"'); +const content = fs.readFileSync(path, { encoding: "utf-8" }); +const updated = content.replace( + /val ensoVersion.*= ".*"/g, + 'val ensoVersion = "' + version + '"' +); fs.writeFileSync(path, updated); console.log("Updated build version to " + version); diff --git a/tools/ci/nightly/clean-old-releases.js b/tools/ci/nightly/clean-old-releases.js index 93996cb6d055..e99c44be04f9 100644 --- a/tools/ci/nightly/clean-old-releases.js +++ b/tools/ci/nightly/clean-old-releases.js @@ -3,32 +3,36 @@ const github = require("./github"); const MAX_NIGHTLY_RELEASES_TO_KEEP = 3; function setProceed(proceed) { - console.log("::set-output name=proceed::" + proceed) + console.log("::set-output name=proceed::" + proceed); } async function main() { const nightlies = await github.fetchNightlies(); if (nightlies.length <= MAX_NIGHTLY_RELEASES_TO_KEEP) { - console.log("No releases to clean.") + console.log("No releases to clean."); return; } const toRemoveCount = nightlies.length - MAX_NIGHTLY_RELEASES_TO_KEEP; if (toRemoveCount > 5) { - console.log("There are " + toRemoveCount + " not-cleaned releases, but every time one is added, older ones are cleaned, so normally there should be at most one to remove. This is very suspicious and it may indicate that the selected set of releases is too broad. Cleaning has been abandoned and the situation should be investigated manually.") + console.log( + "There are " + + toRemoveCount + + " not-cleaned releases, but every time one is added, older ones are cleaned, so normally there should be at most one to remove. This is very suspicious and it may indicate that the selected set of releases is too broad. Cleaning has been abandoned and the situation should be investigated manually." + ); return; } for (let i = MAX_NIGHTLY_RELEASES_TO_KEEP; i < nightlies.length; ++i) { const id = nightlies[i].id; - console.log("Removing old release " + id + ".") + console.log("Removing old release " + id + "."); await github.removeRelease(id); } - console.log("Cleanup complete.") + console.log("Cleanup complete."); } -main().catch(err => { +main().catch((err) => { console.error(err); process.exit(1); }); diff --git a/tools/ci/nightly/extract-release-notes.js b/tools/ci/nightly/extract-release-notes.js index a1d957037722..38cd2e88fe0b 100644 --- a/tools/ci/nightly/extract-release-notes.js +++ b/tools/ci/nightly/extract-release-notes.js @@ -1,4 +1,4 @@ -const fs = require('fs'); +const fs = require("fs"); const inputPath = process.argv[2]; const outputPath = process.argv[3]; @@ -6,8 +6,8 @@ const outputPath = process.argv[3]; console.log("Extracting release notes from " + inputPath + " to " + outputPath); /** Returns the part of the text until the second top-level heading (exclusive) - * in Markdown formatting. - */ + * in Markdown formatting. + */ function cutFirstSection(content) { // TODO [RW] can we assume that '# ' is always the start of a section? // This requires that there are no code snippets with comments starting at the @@ -28,11 +28,14 @@ function cutFirstSection(content) { return content; } const secondHeadingOffsetInContent = restStart + secondHeading; - const firstSectionContent = content.substring(0, secondHeadingOffsetInContent); + const firstSectionContent = content.substring( + 0, + secondHeadingOffsetInContent + ); return firstSectionContent; } -const content = fs.readFileSync(inputPath, {encoding: "utf-8"}); +const content = fs.readFileSync(inputPath, { encoding: "utf-8" }); const nightlyPart = cutFirstSection(content); fs.writeFileSync(outputPath, nightlyPart); diff --git a/tools/ci/nightly/github.js b/tools/ci/nightly/github.js index e8c81f3bd055..a2cf58473793 100644 --- a/tools/ci/nightly/github.js +++ b/tools/ci/nightly/github.js @@ -4,7 +4,8 @@ const octokit = new Octokit(); const organization = "enso-org"; function determineRepositoryName() { const fallback = "enso"; - const fallbackMessage = "Could not determine the repository name, falling back to the default."; + const fallbackMessage = + "Could not determine the repository name, falling back to the default."; const fullName = process.env.GITHUB_REPOSITORY; if (!fullName) { console.log(fallbackMessage); @@ -28,9 +29,9 @@ function isNightly(release) { } async function fetchAllReleases() { - const res = await octokit.request('GET /repos/{owner}/{repo}/releases', { + const res = await octokit.request("GET /repos/{owner}/{repo}/releases", { owner: organization, - repo: repo + repo: repo, }); return res.data; } @@ -42,20 +43,26 @@ async function fetchNightlies() { } async function removeRelease(id) { - return await octokit.request('DELETE /repos/{owner}/{repo}/releases/{release_id}', { - owner: organization, - repo: repo, - release_id: id - }); + return await octokit.request( + "DELETE /repos/{owner}/{repo}/releases/{release_id}", + { + owner: organization, + repo: repo, + release_id: id, + } + ); } async function triggerWorkflow(repo, workflow_id, ref) { - await octokit.request('POST /repos/{owner}/{repo}/actions/workflows/{workflow_id}/dispatches', { - owner: organization, - repo: repo, - workflow_id: workflow_id, - ref: ref - }) + await octokit.request( + "POST /repos/{owner}/{repo}/actions/workflows/{workflow_id}/dispatches", + { + owner: organization, + repo: repo, + workflow_id: workflow_id, + ref: ref, + } + ); } exports.fetchAllReleases = fetchAllReleases; diff --git a/tools/ci/nightly/preflight-check.js b/tools/ci/nightly/preflight-check.js index 771cc31a8b1b..803baca69eb5 100644 --- a/tools/ci/nightly/preflight-check.js +++ b/tools/ci/nightly/preflight-check.js @@ -3,13 +3,15 @@ const github = require("./github"); const currentHeadSha = process.argv[2]; function setProceed(proceed) { - console.log("::set-output name=proceed::" + proceed) + console.log("::set-output name=proceed::" + proceed); } async function main() { const nightlies = await github.fetchNightlies(); if (nightlies.length == 0) { - console.log("No prior nightly releases found. Proceeding with the first release.") + console.log( + "No prior nightly releases found. Proceeding with the first release." + ); setProceed(true); return; } @@ -17,15 +19,25 @@ async function main() { const first = nightlies[0]; const firstNightlySha = first.target_commitish; if (firstNightlySha == currentHeadSha) { - console.log("Current commit (" + currentHeadSha + ") is the same as for the most recent nightly build. A new build is not needed.") + console.log( + "Current commit (" + + currentHeadSha + + ") is the same as for the most recent nightly build. A new build is not needed." + ); setProceed(false); } else { - console.log("Current commit (" + currentHeadSha + ") is different from the most recent nightly build (" + firstNightlySha + "). Proceeding with a new nightly build.") + console.log( + "Current commit (" + + currentHeadSha + + ") is different from the most recent nightly build (" + + firstNightlySha + + "). Proceeding with a new nightly build." + ); setProceed(true); } } -main().catch(err => { +main().catch((err) => { console.error(err); process.exit(1); }); diff --git a/tools/ci/nightly/trigger-workflow.js b/tools/ci/nightly/trigger-workflow.js index cb74e4fbe1a5..067004b4aadc 100644 --- a/tools/ci/nightly/trigger-workflow.js +++ b/tools/ci/nightly/trigger-workflow.js @@ -5,12 +5,14 @@ const workflow_id = process.argv[3]; const ref = process.argv[4]; async function main() { - console.log("Triggering workflow " + workflow_id + " in " + repo + " on " + ref); + console.log( + "Triggering workflow " + workflow_id + " in " + repo + " on " + ref + ); await github.triggerWorkflow(repo, workflow_id, ref); - console.log("Done.") + console.log("Done."); } -main().catch(err => { +main().catch((err) => { console.error(err); process.exit(1); }); From 8f91c382f63490114806fdcd2fa7780d7f94540e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Apr 2021 14:41:56 +0200 Subject: [PATCH 17/30] Add tests --- .../components/LauncherRunnerSpec.scala | 28 +++++++++++++++++-- .../org/enso/loggingservice/TestLogger.scala | 1 + .../RuntimeVersionManagerSpec.scala | 11 +++++++- .../releases/ReleaseProviderException.scala | 8 ++++-- .../engine/EngineReleaseProvider.scala | 11 +++++--- .../releases/github/GithubAPI.scala | 2 +- .../testing/FakeReleaseProvider.scala | 5 ++-- 7 files changed, 53 insertions(+), 13 deletions(-) diff --git a/engine/launcher/src/test/scala/org/enso/launcher/components/LauncherRunnerSpec.scala b/engine/launcher/src/test/scala/org/enso/launcher/components/LauncherRunnerSpec.scala index 6ca723c240ec..ac09f4194fed 100644 --- a/engine/launcher/src/test/scala/org/enso/launcher/components/LauncherRunnerSpec.scala +++ b/engine/launcher/src/test/scala/org/enso/launcher/components/LauncherRunnerSpec.scala @@ -2,7 +2,6 @@ package org.enso.launcher.components import java.nio.file.{Files, Path} import java.util.UUID - import akka.http.scaladsl.model.Uri import nl.gn0s1s.bump.SemVer import org.enso.runtimeversionmanager.FileSystem.PathSyntax @@ -10,7 +9,7 @@ import org.enso.runtimeversionmanager.config.GlobalConfigurationManager import org.enso.runtimeversionmanager.runner._ import org.enso.runtimeversionmanager.test.RuntimeVersionManagerTest import org.enso.launcher.project.ProjectManager -import org.enso.loggingservice.LogLevel +import org.enso.loggingservice.{LogLevel, TestLogger} import scala.concurrent.Future @@ -138,6 +137,31 @@ class LauncherRunnerSpec extends RuntimeVersionManagerTest { commandLine should include(s"--new-project-author-email $authorEmail") } + "warn when creating a project using a nightly version" in { + val runner = makeFakeRunner() + val projectPath = getTestDirectory / "project2" + val nightlyVersion = SemVer(0, 0, 0, Some("SNAPSHOT.2000-01-01")) + val logs = TestLogger.gatherLogs { + runner + .newProject( + path = projectPath, + name = "ProjectName2", + engineVersion = nightlyVersion, + authorName = None, + authorEmail = None, + additionalArguments = Seq() + ) + .get + } + assert( + logs.exists(msg => + msg.logLevel == LogLevel.Warning && msg.message.contains( + "Consider using a stable version." + ) + ) + ) + } + "run repl with default version and additional arguments" in { val runner = makeFakeRunner() val runSettings = runner diff --git a/lib/scala/logging-service/src/main/scala/org/enso/loggingservice/TestLogger.scala b/lib/scala/logging-service/src/main/scala/org/enso/loggingservice/TestLogger.scala index 56f86f0d103f..e05b8ad27dfe 100644 --- a/lib/scala/logging-service/src/main/scala/org/enso/loggingservice/TestLogger.scala +++ b/lib/scala/logging-service/src/main/scala/org/enso/loggingservice/TestLogger.scala @@ -36,6 +36,7 @@ object TestLogger { ) Await.ready(future, 1.second) action + Thread.sleep(100) LoggingServiceManager.tearDown() printer.getLoggedMessages } diff --git a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManagerSpec.scala b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManagerSpec.scala index 269dbf7ddeef..0ac83299d15c 100644 --- a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManagerSpec.scala +++ b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManagerSpec.scala @@ -1,10 +1,10 @@ package org.enso.runtimeversionmanager.components import java.nio.file.{Files, Path} - import nl.gn0s1s.bump.SemVer import org.enso.runtimeversionmanager.FileSystem.PathSyntax import org.enso.runtimeversionmanager.config.GlobalConfigurationManager +import org.enso.runtimeversionmanager.releases.ReleaseNotFound import org.enso.runtimeversionmanager.test.{ RuntimeVersionManagerTest, TestRuntimeVersionManagementUserInterface @@ -112,6 +112,15 @@ class RuntimeVersionManagerSpec extends RuntimeVersionManagerTest with OsSpec { componentsManager.findEngine(brokenVersion).value } + "issue a context-specific error when a nightly release cannot be found" in { + val componentsManager = makeManagers()._2 + val nightlyVersion = SemVer(0, 0, 0, Some("SNAPSHOT.2000-01-01")) + val exception = intercept[ReleaseNotFound] { + componentsManager.findOrInstallEngine(nightlyVersion) + } + exception.getMessage should include("Nightly releases expire") + } + "uninstall the runtime iff it is not used by any engines" in { val componentsManager = makeRuntimeVersionManager() val engineVersions = diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/ReleaseProviderException.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/ReleaseProviderException.scala index 7d742c5b8dc1..34eccc58dde2 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/ReleaseProviderException.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/ReleaseProviderException.scala @@ -27,5 +27,9 @@ object ReleaseProviderException { */ case class ReleaseNotFound( tag: String, - cause: Throwable = null -) extends ReleaseProviderException(s"Cannot find release `$tag`.", cause) + message: Option[String] = None, + cause: Throwable = null +) extends ReleaseProviderException( + message.getOrElse(s"Cannot find release `$tag`."), + cause + ) diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/engine/EngineReleaseProvider.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/engine/EngineReleaseProvider.scala index 373a585556e5..346f2383cfd8 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/engine/EngineReleaseProvider.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/engine/EngineReleaseProvider.scala @@ -57,14 +57,17 @@ class EngineReleaseProvider(releaseProvider: SimpleReleaseProvider) * releases if applicable. */ private def wrapFetchError[A](result: Try[A]): Try[A] = result.recoverWith { - case ReleaseNotFound(tag, cause) if tag.contains(Engine.nightlyInfix) => + case ReleaseNotFound(tag, _, cause) if tag.contains(Engine.nightlyInfix) => // TODO [RW] shouldn't we somehow explain *how* to upgrade? Ideally the // launcher should include a command for that. Failure( ReleaseNotFound( - s"Cannot find release `$tag`. Nightly releases expire after some " + - s"time. Consider upgrading to a stable release or a newer nightly " + - s"build.", + tag, + Some( + s"Cannot find release `$tag`. Nightly releases expire after some " + + s"time. Consider upgrading to a stable release or a newer " + + s"nightly build." + ), cause ) ) diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/github/GithubAPI.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/github/GithubAPI.scala index c8cbdca83266..94ed977a83bf 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/github/GithubAPI.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/github/GithubAPI.scala @@ -109,7 +109,7 @@ object GithubAPI { .map(err => handleError( response, - ReleaseNotFound(tag, err) + ReleaseNotFound(tag, cause = err) ) ) .toTry diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/testing/FakeReleaseProvider.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/testing/FakeReleaseProvider.scala index f066cd70d6a6..7a4b6cd8e91d 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/testing/FakeReleaseProvider.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/testing/FakeReleaseProvider.scala @@ -1,14 +1,13 @@ package org.enso.runtimeversionmanager.releases.testing import java.nio.file.{Files, Path, StandardCopyOption} - import org.enso.cli.task.{ProgressListener, TaskProgress} import org.enso.runtimeversionmanager.FileSystem import org.enso.runtimeversionmanager.locking.{LockManager, LockType} import org.enso.runtimeversionmanager.releases.{ Asset, Release, - ReleaseProviderException, + ReleaseNotFound, SimpleReleaseProvider } @@ -39,7 +38,7 @@ case class FakeReleaseProvider( override def releaseForTag(tag: String): Try[Release] = releases .find(_.tag == tag) - .toRight(ReleaseProviderException(s"Release $tag does not exist.")) + .toRight(ReleaseNotFound(tag)) .toTry /** @inheritdoc */ From 394b2562c5f240ddd630e5e087dabc6ad8a3f47d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Apr 2021 14:45:51 +0200 Subject: [PATCH 18/30] Remove debug trigger --- .github/workflows/nightly.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 3d2691878a74..25845b2316a4 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -21,20 +21,21 @@ jobs: name: Check If The Build Should Proceed runs-on: ubuntu-18.04 timeout-minutes: 10 - # That true here is for debugging on staging - if: > - ${{ github.event_name == 'schedule' || - contains(github.event.head_commit.message,'[release: nightly]') || true }} + if: + "${{ github.event_name == 'schedule' || + contains(github.event.head_commit.message,'[release: nightly]') }}" outputs: proceed: ${{ steps.check-assumptions.outputs.proceed }} nightly-version: ${{ steps.compute-version.outputs.nightly-version }} steps: - name: Checkout uses: actions/checkout@v2 - - name: Debug Print commit message + - name: Debug Print shell: bash run: | echo ${{ github.event.head_commit.message }} + echo ${{ github.event_name }} + echo ${{ github.event_name == 'schedule' || contains(github.event.head_commit.message,'[release: nightly]') }} - id: check-assumptions name: Check If The Build Should Proceed shell: bash From 66804ba9effadd6452547971a970250a603f7ef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Apr 2021 15:11:07 +0200 Subject: [PATCH 19/30] Formatting --- tools/ci/check-changelog.js | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tools/ci/check-changelog.js b/tools/ci/check-changelog.js index 4e364ab82c6c..d036499e1a02 100644 --- a/tools/ci/check-changelog.js +++ b/tools/ci/check-changelog.js @@ -1,7 +1,7 @@ -const fs = require('fs'); -const proc = require('child_process'); +const fs = require("fs"); +const proc = require("child_process"); -const skipChangelogInfix = "[no-changelog]" +const skipChangelogInfix = "[no-changelog]"; const changelogPath = process.argv[2]; const baseRef = process.argv[3]; @@ -13,7 +13,7 @@ function wasChangelogModified() { } function isChangelogSkipped() { - const logArgs = ["--no-pager", "log", "HEAD~3...HEAD", "--pretty=oneline"] + const logArgs = ["--no-pager", "log", "HEAD~3...HEAD", "--pretty=oneline"]; const output = proc.spawnSync("git", diffArgs).stdout.toString("utf-8"); const containsSkipCommit = output.indexOf(skipChangelogInfix) >= 0; return containsSkipCommit; @@ -25,7 +25,11 @@ if (wasChangelogModified()) { } else { console.log("No changes to the changelog"); if (isChangelogSkipped()) { - console.log("But one of the commits within the PR includes " + skipChangelogInfix + ", so the check is skipped."); + console.log( + "But one of the commits within the PR includes " + + skipChangelogInfix + + ", so the check is skipped." + ); process.exit(0); } else { process.exit(1); From c21694a7e71d9da19ab05d3dd2a29a86db1d1319 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Apr 2021 16:46:12 +0200 Subject: [PATCH 20/30] Fix changelog diff --- .github/workflows/docs.yml | 2 +- tools/ci/check-changelog.js | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7bc73b5ca6e4..28af4f3521b0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -45,4 +45,4 @@ jobs: - name: Check If Changelog Was Modified shell: bash run: | - node tools/ci/check-changelog.js RELEASES.md ${{ github.base_ref }} + node tools/ci/check-changelog.js RELEASES.md ${{ github.event.pull_request.base.sha }} diff --git a/tools/ci/check-changelog.js b/tools/ci/check-changelog.js index d036499e1a02..2679b2caf60a 100644 --- a/tools/ci/check-changelog.js +++ b/tools/ci/check-changelog.js @@ -6,15 +6,24 @@ const changelogPath = process.argv[2]; const baseRef = process.argv[3]; function wasChangelogModified() { - const diffArgs = ["--no-pager", "diff", "--exit-code", ref, changelogPath]; - const exitCode = proc.spawnSync("git", diffArgs).status; + const diffArgs = [ + "--no-pager", + "diff", + "--exit-code", + baseRef, + "--", + changelogPath, + ]; + const result = proc.spawnSync("git", diffArgs); + const exitCode = result.status; + console.log(result.stdout.toString("utf-8")); const noDifference = exitCode == 0; return !noDifference; } function isChangelogSkipped() { const logArgs = ["--no-pager", "log", "HEAD~3...HEAD", "--pretty=oneline"]; - const output = proc.spawnSync("git", diffArgs).stdout.toString("utf-8"); + const output = proc.spawnSync("git", logArgs).stdout.toString("utf-8"); const containsSkipCommit = output.indexOf(skipChangelogInfix) >= 0; return containsSkipCommit; } From a372fc0723f49624c71f44d1615745c8a785dd68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 21 Apr 2021 17:32:46 +0200 Subject: [PATCH 21/30] WIP docs --- .github/workflows/docs.yml | 1 - docs/distribution/README.md | 2 ++ docs/distribution/nightly.md | 38 ++++++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 docs/distribution/nightly.md diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 28af4f3521b0..a7378cec9e83 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -41,7 +41,6 @@ jobs: uses: actions/setup-node@v1 with: node-version: ${{ env.nodeVersion }} - # TODO how to indicate that we don't want to check? Is it enough that any commit of the PR has `no-changelog` ? - name: Check If Changelog Was Modified shell: bash run: | diff --git a/docs/distribution/README.md b/docs/distribution/README.md index d2d5841fa3de..cb3e40469526 100644 --- a/docs/distribution/README.md +++ b/docs/distribution/README.md @@ -31,3 +31,5 @@ dependencies, and Enso projects for use by our users. standard libraries for Enso. - [**Bundles**](bundles.md) An explanation of distributed bundles that contain all components necessary to run Enso out of the box. +- [**Nightly**](nightly.md) Description of the infrastructure related to the + nightly builds. diff --git a/docs/distribution/nightly.md b/docs/distribution/nightly.md new file mode 100644 index 000000000000..80ad36be086f --- /dev/null +++ b/docs/distribution/nightly.md @@ -0,0 +1,38 @@ +--- +layout: developer-doc +title: Nightly Builds +category: distribution +tags: [distribution, release, nightly] +order: 10 +--- + +# Nightly Builds + +This document describes the infrastructure for nightly builds. + + + +- TODO + + + +## Scheduled Builds + +After each weekday, at 4am UTC a scheduled workflow prepares a release from the latest commit on the `main` branch. + +A new release is not made if there were no new commits to `main` since the previous release. + +Each nightly build gets the current date appended to the version and since commits on `main` should have versions marked as SNAPSHOT (for example `1.2.3-SNAPSHOT`), the nightly release version will then look like `1.2.3-SNAPSHOT.1970-01-01`. + +## Manual Builds + +It is possible to trigger a nightly release manually by adding `[release: nightly]` to the commit message. + +TODO: which branches this works on, handling collisions + +## Release Retention + +Only the 3 most recent nightly releases are kept. When a nightly release finishes building it removes any older releases to keep only the most recent 3. + +Thus, setting a nightly version for a project will require updating it very soon (currenty after about 3 days). The launcher issues a warning when creating a project with a nightly version and if the version is not available it explains that this may be due to the retention policy. + From e9d689f524364aaea28f1c5405ddff5415c2d9ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Tue, 27 Apr 2021 17:37:30 +0200 Subject: [PATCH 22/30] Add concurrency and filter branches for nightly pushes --- .github/workflows/nightly.yml | 4 ++++ .github/workflows/release.yml | 2 ++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 25845b2316a4..609af90c6e6b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -3,6 +3,8 @@ name: Nightly Release CI on: push: + branches: + - main schedule: - cron: "0 4 * * 2-6" # 4am from Tuesday to Saturday (i.e. after every workday) @@ -16,6 +18,8 @@ env: # Please ensure that this is in sync with rustVersion in build.sbt rustToolchain: nightly-2019-11-04 +concurrency: "releases" + jobs: preflight-check: name: Check If The Build Should Proceed diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2d50e353d10d..286e24b54b1f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,6 +15,8 @@ env: # Please ensure that this is in sync with rustVersion in build.sbt rustToolchain: nightly-2019-11-04 +concurrency: "releases" + jobs: # This job should be kept up-to-date with scala.yml#build (but keep the added version check) build: From 0d6ff9dffc7a88263de9e62e18286e01cee299ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Tue, 27 Apr 2021 18:30:30 +0200 Subject: [PATCH 23/30] Compute version in preflight-check to account for multiple releases on a single day --- .github/workflows/nightly.yml | 17 ++---- tools/ci/nightly/preflight-check.js | 91 +++++++++++++++++++++++++++-- 2 files changed, 90 insertions(+), 18 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 609af90c6e6b..b37ae1d128c6 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -29,8 +29,8 @@ jobs: "${{ github.event_name == 'schedule' || contains(github.event.head_commit.message,'[release: nightly]') }}" outputs: - proceed: ${{ steps.check-assumptions.outputs.proceed }} - nightly-version: ${{ steps.compute-version.outputs.nightly-version }} + proceed: ${{ steps.preparations.outputs.proceed }} + nightly-version: ${{ steps.preparations.outputs.nightly-version }} steps: - name: Checkout uses: actions/checkout@v2 @@ -40,24 +40,17 @@ jobs: echo ${{ github.event.head_commit.message }} echo ${{ github.event_name }} echo ${{ github.event_name == 'schedule' || contains(github.event.head_commit.message,'[release: nightly]') }} - - id: check-assumptions - name: Check If The Build Should Proceed + - id: preparations + name: Check If The Build Should Proceed And Prepare Version String shell: bash + working-directory: repo/tools/ci/nightly env: GITHUB_TOKEN: ${{ github.token }} # The script below sets an output 'proceed' to true or false depending on whether the nightly build should proceed. # Nightly builds are skipped if no new changes are present since the last one. run: | - cd tools/ci/nightly/ npm install node preflight-check.js ${{ github.sha }} - - name: Compute Version - id: compute-version - if: ${{ steps.check-assumptions.outputs.proceed == 'true' }} - shell: bash - run: | - VERSION=$(cat build.sbt | grep 'val ensoVersion' | sed 's/.*"\([^)]*\)".*/\1/').$(date --iso-8601) - echo ::set-output name=nightly-version::$VERSION # This job should be kept up-to-date with scala.yml#build (but keep the relevant changes) build: diff --git a/tools/ci/nightly/preflight-check.js b/tools/ci/nightly/preflight-check.js index 803baca69eb5..ad34f6179d0f 100644 --- a/tools/ci/nightly/preflight-check.js +++ b/tools/ci/nightly/preflight-check.js @@ -1,19 +1,43 @@ +const fs = require("fs"); const github = require("./github"); const currentHeadSha = process.argv[2]; +const buildConfigPath = "../../../build.sbt"; +/// Returns the current date formatted as 'YYYY-mm-dd'. +function isoDate() { + const now = new Date(); + const year = "" + now.getFullYear(); + let month = "" + (now.getMonth() + 1); + let day = "" + now.getDate(); + if (month.length < 2) { + month = "0" + month; + } + if (day.length < 2) { + day = "0" + day; + } + return year + "-" + month + "-" + day; +} + +/// Sets the step output 'proceed'. function setProceed(proceed) { console.log("::set-output name=proceed::" + proceed); } -async function main() { - const nightlies = await github.fetchNightlies(); +/// Sets the step output 'nightly-version'. +function setVersionString(name) { + console.log("::set-output name=nightly-version::" + name); +} + +/** Checks if there are any new changes to see if the nightly build should + * proceed. + */ +function checkProceed(nightlies) { if (nightlies.length == 0) { console.log( "No prior nightly releases found. Proceeding with the first release." ); - setProceed(true); - return; + return true; } const first = nightlies[0]; @@ -24,7 +48,7 @@ async function main() { currentHeadSha + ") is the same as for the most recent nightly build. A new build is not needed." ); - setProceed(false); + return false; } else { console.log( "Current commit (" + @@ -33,7 +57,62 @@ async function main() { firstNightlySha + "). Proceeding with a new nightly build." ); - setProceed(true); + return true; + } +} + +/** Prepares a version string for the nightly build. + * + * A '-SNAPSHOT' suffix is added if it is not already present, next the current + * date is appended. If this is not the first nightly build on that date, an + * increasing numeric suffix is added. + */ +function prepareVersionString(nightlies) { + function isTaken(name) { + const tagName = "enso-" + name; + return nightlies.some((entry) => entry.tag_name == tagName); + } + + const content = fs.readFileSync(buildConfigPath, { encoding: "utf-8" }); + const match = content.match(/val ensoVersion += +"(.*)"/); + if (!match) { + console.error("Could not find the version string in configuration!"); + process.exit(1); + } + + const version = match[1]; + let baseName = version; + if (!baseName.endsWith("SNAPSHOT")) { + baseName += "-SNAPSHOT"; + } + + baseName += "." + isoDate(); + + function makeName(ix) { + if (ix == 0) { + return baseName; + } else { + return baseName + "." + ix; + } + } + + let ix = 0; + while (isTaken(makeName(ix))) { + ix++; + } + + const name = makeName(ix); + console.log("The build will be using version '" + name + "'"); + return name; +} + +async function main() { + const nightlies = await github.fetchNightlies(); + const shouldProceed = checkProceed(nightlies); + setProceed(shouldProceed); + if (shouldProceed) { + const versionString = prepareVersionString(nightlies); + setVersionString(versionString); } } From d96d38a25f5a4e956369962bd5d1e30390899615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Tue, 27 Apr 2021 18:32:18 +0200 Subject: [PATCH 24/30] Try an action for removing old releases --- .github/workflows/nightly.yml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b37ae1d128c6..bd124789fc89 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -43,7 +43,7 @@ jobs: - id: preparations name: Check If The Build Should Proceed And Prepare Version String shell: bash - working-directory: repo/tools/ci/nightly + working-directory: tools/ci/nightly env: GITHUB_TOKEN: ${{ github.token }} # The script below sets an output 'proceed' to true or false depending on whether the nightly build should proceed. @@ -542,9 +542,17 @@ jobs: # run: node trigger-workflow.js ide gui-ci develop # env: # GITHUB_TOKEN: ${{ github.token }} - - name: Remove Old Releases - shell: bash - working-directory: repo/tools/ci/nightly - run: node clean-old-releases.js +# - name: Remove Old Releases +# shell: bash +# working-directory: repo/tools/ci/nightly +# run: node clean-old-releases.js +# env: +# GITHUB_TOKEN: ${{ github.token }} + - uses: dev-drprasad/delete-older-releases@v0.2.0 + name: Remove Old Releases + with: + keep_latest: 3 + delete_tag_pattern: SNAPSHOT + delete_tags: true env: - GITHUB_TOKEN: ${{ github.token }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 14b86f16f72b24efa7942fb4f34100e46478ee97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 28 Apr 2021 14:54:41 +0200 Subject: [PATCH 25/30] Fix auth, change release notes and verify them --- .github/workflows/docs.yml | 9 ++++ .github/workflows/nightly.yml | 14 +++--- RELEASES.md | 2 +- tools/ci/nightly/clean-old-releases.js | 38 -------------- tools/ci/nightly/extract-release-notes.js | 60 +++++++++++++++-------- tools/ci/nightly/github.js | 25 +++++----- tools/ci/nightly/publish-release.js | 14 ++++++ 7 files changed, 83 insertions(+), 79 deletions(-) delete mode 100644 tools/ci/nightly/clean-old-releases.js create mode 100644 tools/ci/nightly/publish-release.js diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a7378cec9e83..84d069468224 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -41,7 +41,16 @@ jobs: uses: actions/setup-node@v1 with: node-version: ${{ env.nodeVersion }} + - name: Install Dependencies + shell: bash + working-directory: tools/ci/nightly + run: | + npm install - name: Check If Changelog Was Modified shell: bash run: | node tools/ci/check-changelog.js RELEASES.md ${{ github.event.pull_request.base.sha }} + - name: Verify That Changelog Format Is Correct + shell: bash + run: | + node tools/ci/nightly/extract-release-notes.js RELEASES.md release_notes.md diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index bd124789fc89..ef9d1ebbd003 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -373,7 +373,7 @@ jobs: tag_name: enso-${{ env.DIST_VERSION }} release_name: Enso Nightly ${{ env.DIST_VERSION }} body_path: repo/release_notes.md - draft: false + draft: true prerelease: true # Upload the assets to the created release @@ -534,6 +534,12 @@ jobs: asset_path: repo/built-distribution/launcher-manifest/launcher-manifest.yaml asset_name: launcher-manifest.yaml asset_content_type: application/yaml + - name: Publish Release + shell: bash + working-directory: repo/tools/ci/nightly + run: node publish-release.js ${{ steps.create_release.outputs.id }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # TODO enable this once IDE supports triggers # - name: Trigger IDE Build @@ -542,12 +548,6 @@ jobs: # run: node trigger-workflow.js ide gui-ci develop # env: # GITHUB_TOKEN: ${{ github.token }} -# - name: Remove Old Releases -# shell: bash -# working-directory: repo/tools/ci/nightly -# run: node clean-old-releases.js -# env: -# GITHUB_TOKEN: ${{ github.token }} - uses: dev-drprasad/delete-older-releases@v0.2.0 name: Remove Old Releases with: diff --git a/RELEASES.md b/RELEASES.md index 4153be6890f8..b15a22de091f 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,4 +1,4 @@ -# Nightly +# Enso Next ## Miscellaneous diff --git a/tools/ci/nightly/clean-old-releases.js b/tools/ci/nightly/clean-old-releases.js deleted file mode 100644 index e99c44be04f9..000000000000 --- a/tools/ci/nightly/clean-old-releases.js +++ /dev/null @@ -1,38 +0,0 @@ -const github = require("./github"); - -const MAX_NIGHTLY_RELEASES_TO_KEEP = 3; - -function setProceed(proceed) { - console.log("::set-output name=proceed::" + proceed); -} - -async function main() { - const nightlies = await github.fetchNightlies(); - if (nightlies.length <= MAX_NIGHTLY_RELEASES_TO_KEEP) { - console.log("No releases to clean."); - return; - } - - const toRemoveCount = nightlies.length - MAX_NIGHTLY_RELEASES_TO_KEEP; - if (toRemoveCount > 5) { - console.log( - "There are " + - toRemoveCount + - " not-cleaned releases, but every time one is added, older ones are cleaned, so normally there should be at most one to remove. This is very suspicious and it may indicate that the selected set of releases is too broad. Cleaning has been abandoned and the situation should be investigated manually." - ); - return; - } - - for (let i = MAX_NIGHTLY_RELEASES_TO_KEEP; i < nightlies.length; ++i) { - const id = nightlies[i].id; - console.log("Removing old release " + id + "."); - await github.removeRelease(id); - } - - console.log("Cleanup complete."); -} - -main().catch((err) => { - console.error(err); - process.exit(1); -}); diff --git a/tools/ci/nightly/extract-release-notes.js b/tools/ci/nightly/extract-release-notes.js index 38cd2e88fe0b..706aaf556d96 100644 --- a/tools/ci/nightly/extract-release-notes.js +++ b/tools/ci/nightly/extract-release-notes.js @@ -2,6 +2,7 @@ const fs = require("fs"); const inputPath = process.argv[2]; const outputPath = process.argv[3]; +const version = process.env.DIST_VERSION; console.log("Extracting release notes from " + inputPath + " to " + outputPath); @@ -9,35 +10,54 @@ console.log("Extracting release notes from " + inputPath + " to " + outputPath); * in Markdown formatting. */ function cutFirstSection(content) { - // TODO [RW] can we assume that '# ' is always the start of a section? - // This requires that there are no code snippets with comments starting at the - // beginning of the line. - function findSectionStart(text) { - return text.search(/^# /gm); + const nightlySectionRegex = /^# Enso Next$/gm; + function findNightlySectionStart(text) { + return text.search(nightlySectionRegex); + } + const regularSectionRegex = /^# Enso .*? \(\d+-\d+-\d+\)$/gm; + function findFirstRegularSectionStart(text) { + return text.search(regularSectionRegex); + } + function findNewline(text) { + return text.indexOf("\n"); } - const firstHeading = findSectionStart(content); + const firstHeading = findNightlySectionStart(content); if (firstHeading < 0) { - throw "No sections in file!"; + throw "Could not find the nightly section, matching " + nightlySectionRegex; } - const restStart = firstHeading + 2; + + const restOffset = firstHeading + 2; + const newLineOffset = findNewline(content.substring(restOffset)); + if (newLineOffset < 0) { + throw "No content after the section heading" + } + const restStart = restOffset + newLineOffset + 1; + const rest = content.substring(restStart); - const secondHeading = findSectionStart(rest); + const secondHeading = findFirstRegularSectionStart(rest); if (secondHeading < 0) { - // There is only one section. - return content; + throw "Could not find the first released section, matching" + regularSectionRegex; } - const secondHeadingOffsetInContent = restStart + secondHeading; - const firstSectionContent = content.substring( + + const firstSectionContent = rest.substring( 0, - secondHeadingOffsetInContent + secondHeading ); - return firstSectionContent; + + const firstSectionHeader = "# Enso Nightly " + version + "\n"; + + return firstSectionHeader + firstSectionContent; } -const content = fs.readFileSync(inputPath, { encoding: "utf-8" }); -const nightlyPart = cutFirstSection(content); -fs.writeFileSync(outputPath, nightlyPart); +try { + const content = fs.readFileSync(inputPath, { encoding: "utf-8" }); + const nightlyPart = cutFirstSection(content); + fs.writeFileSync(outputPath, nightlyPart); -console.log("Created " + outputPath + " with the following content:"); -console.log(nightlyPart); + console.log("Created " + outputPath + " with the following content:"); + console.log(nightlyPart); +} catch (exc) { + console.error(exc); + process.exit(1); +} diff --git a/tools/ci/nightly/github.js b/tools/ci/nightly/github.js index a2cf58473793..b758c62063da 100644 --- a/tools/ci/nightly/github.js +++ b/tools/ci/nightly/github.js @@ -1,5 +1,4 @@ const { Octokit } = require("@octokit/core"); -const octokit = new Octokit(); const organization = "enso-org"; function determineRepositoryName() { @@ -20,8 +19,10 @@ function determineRepositoryName() { return fallback; } } + const repo = determineRepositoryName(); const token = process.env.GITHUB_TOKEN; +const octokit = new Octokit({auth: token}); function isNightly(release) { const nightlyInfix = "Nightly"; @@ -42,17 +43,6 @@ async function fetchNightlies() { return nightlies; } -async function removeRelease(id) { - return await octokit.request( - "DELETE /repos/{owner}/{repo}/releases/{release_id}", - { - owner: organization, - repo: repo, - release_id: id, - } - ); -} - async function triggerWorkflow(repo, workflow_id, ref) { await octokit.request( "POST /repos/{owner}/{repo}/actions/workflows/{workflow_id}/dispatches", @@ -65,7 +55,16 @@ async function triggerWorkflow(repo, workflow_id, ref) { ); } +async function publishRelease(id) { + return await octokit.request('PATCH /repos/{owner}/{repo}/releases/{release_id}', { + owner: organization, + repo: repo, + release_id: id, + draft: false + }); +} + exports.fetchAllReleases = fetchAllReleases; exports.fetchNightlies = fetchNightlies; -exports.removeRelease = removeRelease; +exports.publishRelease = publishRelease; exports.repository = repo; diff --git a/tools/ci/nightly/publish-release.js b/tools/ci/nightly/publish-release.js new file mode 100644 index 000000000000..fdcb2b35dc9d --- /dev/null +++ b/tools/ci/nightly/publish-release.js @@ -0,0 +1,14 @@ +const github = require("./github"); + +const releaseId = process.argv[2]; + +async function main() { + console.log("Making release " + releaseId + " public."); + await github.publishRelease(releaseId); + console.log("Done.") +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); From 78815a43d611e329def7dfceac66d92c6649d262 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 28 Apr 2021 18:21:52 +0200 Subject: [PATCH 26/30] Formatting --- RELEASES.md | 3 ++- tools/ci/nightly/extract-release-notes.js | 12 ++++++------ tools/ci/nightly/github.js | 17 ++++++++++------- tools/ci/nightly/preflight-check.js | 14 +++++++------- tools/ci/nightly/publish-release.js | 4 ++-- 5 files changed, 27 insertions(+), 23 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index b15a22de091f..28ffdf79dca7 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -2,7 +2,8 @@ ## Miscellaneous -- Adding a pipeline for automatic nightly builds. +- Adding a pipeline for automatic nightly builds + ([#1689](https://github.com/enso-org/enso/pull/1689)). # Enso 0.2.11 (2021-04-28) diff --git a/tools/ci/nightly/extract-release-notes.js b/tools/ci/nightly/extract-release-notes.js index 706aaf556d96..22bff26a3c49 100644 --- a/tools/ci/nightly/extract-release-notes.js +++ b/tools/ci/nightly/extract-release-notes.js @@ -30,20 +30,20 @@ function cutFirstSection(content) { const restOffset = firstHeading + 2; const newLineOffset = findNewline(content.substring(restOffset)); if (newLineOffset < 0) { - throw "No content after the section heading" + throw "No content after the section heading"; } const restStart = restOffset + newLineOffset + 1; const rest = content.substring(restStart); const secondHeading = findFirstRegularSectionStart(rest); if (secondHeading < 0) { - throw "Could not find the first released section, matching" + regularSectionRegex; + throw ( + "Could not find the first released section, matching" + + regularSectionRegex + ); } - const firstSectionContent = rest.substring( - 0, - secondHeading - ); + const firstSectionContent = rest.substring(0, secondHeading); const firstSectionHeader = "# Enso Nightly " + version + "\n"; diff --git a/tools/ci/nightly/github.js b/tools/ci/nightly/github.js index b758c62063da..b40e201d6354 100644 --- a/tools/ci/nightly/github.js +++ b/tools/ci/nightly/github.js @@ -22,7 +22,7 @@ function determineRepositoryName() { const repo = determineRepositoryName(); const token = process.env.GITHUB_TOKEN; -const octokit = new Octokit({auth: token}); +const octokit = new Octokit({ auth: token }); function isNightly(release) { const nightlyInfix = "Nightly"; @@ -56,12 +56,15 @@ async function triggerWorkflow(repo, workflow_id, ref) { } async function publishRelease(id) { - return await octokit.request('PATCH /repos/{owner}/{repo}/releases/{release_id}', { - owner: organization, - repo: repo, - release_id: id, - draft: false - }); + return await octokit.request( + "PATCH /repos/{owner}/{repo}/releases/{release_id}", + { + owner: organization, + repo: repo, + release_id: id, + draft: false, + } + ); } exports.fetchAllReleases = fetchAllReleases; diff --git a/tools/ci/nightly/preflight-check.js b/tools/ci/nightly/preflight-check.js index ad34f6179d0f..9bb967a85ea2 100644 --- a/tools/ci/nightly/preflight-check.js +++ b/tools/ci/nightly/preflight-check.js @@ -30,8 +30,8 @@ function setVersionString(name) { } /** Checks if there are any new changes to see if the nightly build should - * proceed. - */ + * proceed. + */ function checkProceed(nightlies) { if (nightlies.length == 0) { console.log( @@ -62,11 +62,11 @@ function checkProceed(nightlies) { } /** Prepares a version string for the nightly build. - * - * A '-SNAPSHOT' suffix is added if it is not already present, next the current - * date is appended. If this is not the first nightly build on that date, an - * increasing numeric suffix is added. - */ + * + * A '-SNAPSHOT' suffix is added if it is not already present, next the current + * date is appended. If this is not the first nightly build on that date, an + * increasing numeric suffix is added. + */ function prepareVersionString(nightlies) { function isTaken(name) { const tagName = "enso-" + name; diff --git a/tools/ci/nightly/publish-release.js b/tools/ci/nightly/publish-release.js index fdcb2b35dc9d..c8edfb1b3020 100644 --- a/tools/ci/nightly/publish-release.js +++ b/tools/ci/nightly/publish-release.js @@ -5,10 +5,10 @@ const releaseId = process.argv[2]; async function main() { console.log("Making release " + releaseId + " public."); await github.publishRelease(releaseId); - console.log("Done.") + console.log("Done."); } -main().catch(err => { +main().catch((err) => { console.error(err); process.exit(1); }); From 3a792096f92ccff0b6746f6a24fcc4eba72bbb9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 28 Apr 2021 18:58:52 +0200 Subject: [PATCH 27/30] Docs --- docs/distribution/nightly.md | 68 +++++++++++++++---- .../components/Engine.scala | 2 +- .../runtimeversionmanager/runner/Runner.scala | 2 +- 3 files changed, 58 insertions(+), 14 deletions(-) diff --git a/docs/distribution/nightly.md b/docs/distribution/nightly.md index 80ad36be086f..ef62d5b2ed02 100644 --- a/docs/distribution/nightly.md +++ b/docs/distribution/nightly.md @@ -8,31 +8,75 @@ order: 10 # Nightly Builds -This document describes the infrastructure for nightly builds. +This document describes the infrastructure for automated nightly builds. -- TODO +- [Triggering the Build](#triggering-the-build) +- [Nightly Build Versions](#nightly-build-versions) +- [Release Notes](#release-notes) -## Scheduled Builds +## Triggering the Build -After each weekday, at 4am UTC a scheduled workflow prepares a release from the latest commit on the `main` branch. +The build may be triggered on two possible events: -A new release is not made if there were no new commits to `main` since the previous release. +- automatically, at 4am UTC after each working day (that is, on Tuesday to + Saturday), +- manually, if a commit with message containing `[release: nightly]` is pushed + to the `main` branch. -Each nightly build gets the current date appended to the version and since commits on `main` should have versions marked as SNAPSHOT (for example `1.2.3-SNAPSHOT`), the nightly release version will then look like `1.2.3-SNAPSHOT.1970-01-01`. +The nightly build is based off of the state of the `main` branch at the moment +when it was triggered. -## Manual Builds +However, when a nightly build is triggered (by any of the two above conditions), +it will only proceed if there are any changes. That is, if the current commit is +the same as the one used for the previous nightly build, the build will not +proceed, because there are no changes. -It is possible to trigger a nightly release manually by adding `[release: nightly]` to the commit message. +Thanks to +[GitHub's concurrency settings](https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency), +only one nightly or regular release job may be running at a given time. If a +build is triggered when another one is in progress, it will be pending until the +first one finishes. -TODO: which branches this works on, handling collisions +## Nightly Build Versions -## Release Retention +The nightly build will modify the version (which is set in `build.sbt`) to +indicate that this is a specific nightly build. -Only the 3 most recent nightly releases are kept. When a nightly release finishes building it removes any older releases to keep only the most recent 3. +If the version does not include a `SNAPSHOT` suffix, it is added. Then the +current date is appended. Moreover, if the build is not the first nightly build +to be done on a given day, an increasing numeric suffix is appended, to +differentiate between builds from a single day. -Thus, setting a nightly version for a project will require updating it very soon (currenty after about 3 days). The launcher issues a warning when creating a project with a nightly version and if the version is not available it explains that this may be due to the retention policy. +For example, if `build.sbt` specifies the version to be `1.2.3` or +`1.2.3-SNAPSHOT`, the nightly build based off of that version, triggered on 1st +of February 2021 will result in version `1.2.3-SNAPSHOT.2021-02-01`. If a +subsequent build based off of the same version is triggered on the same day it +will be `1.2.3-SNAPSHOT.2021-02-01.1` etc. +Only the 3 most recent nightly builds are kept in the repository, any older +builds are removed from releases. + +## Release Notes + +Each PR should update the first, `Enso Next`, section in `RELEASES.md`. These +changes will be later moved to the specific section for the full release. This +section is also used to fill the release notes for the nightly builds. + +Most PRs should update the release notes, so there is a PR check that ensures +the file was modified. However in some situations there is no need to update the +notes. If any commit included as part of a PR includes `[no-changelog]` within +its message, that check is ignored. + +The changelog should keep consistent formatting: + +- each version should be delimited by a top-level section (`#` in Markdown), +- the first section should always be called `Enso Next`, +- all subsequent sections should be called `Enso ()`. + +A heuristic check is ran for PRs, checking that at least the first two sections +satisfy the above requirements - which is necessary for the nightly build +pipeline to be able to correctly infer the release notes. diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Engine.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Engine.scala index 830f28716296..61c284b356c5 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Engine.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Engine.scala @@ -70,7 +70,7 @@ case class Engine(version: SemVer, path: Path, manifest: Manifest) { /** Specifies if the engine version comes from a nightly release. * - * See `docs/distribution/nightly.md` for more information. TODO [RW] docs + * See `docs/distribution/nightly.md` for more information. */ def isNightly: Boolean = Engine.isNightly(version) } diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/runner/Runner.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/runner/Runner.scala index d00292aea905..c3f59e195a99 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/runner/Runner.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/runner/Runner.scala @@ -54,7 +54,7 @@ class Runner( "--new-project-name", name ) ++ authorNameOption ++ authorEmailOption ++ additionalArguments - // TODO [RW] this warning will not be clearly visible in the IDE, do we want some other mechanism there? It would most likely require modifying the protocol. + // TODO [RW] reporting warnings to the IDE (#1710) if (Engine.isNightly(engineVersion)) { Logger[Runner].warn( "Creating a new project using a nightly build. Nightly builds may " + From 1bf1a8f6168d18d6b4683278519268c21cd8dc6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 30 Apr 2021 17:00:22 +0200 Subject: [PATCH 28/30] Add a check to changelog --- tools/ci/check-changelog.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/ci/check-changelog.js b/tools/ci/check-changelog.js index 2679b2caf60a..1c5fcad56798 100644 --- a/tools/ci/check-changelog.js +++ b/tools/ci/check-changelog.js @@ -15,6 +15,10 @@ function wasChangelogModified() { changelogPath, ]; const result = proc.spawnSync("git", diffArgs); + if (result.error) { + console.log("Cannot access git", result.error); + process.exit(1); + } const exitCode = result.status; console.log(result.stdout.toString("utf-8")); const noDifference = exitCode == 0; @@ -23,7 +27,13 @@ function wasChangelogModified() { function isChangelogSkipped() { const logArgs = ["--no-pager", "log", "HEAD~3...HEAD", "--pretty=oneline"]; - const output = proc.spawnSync("git", logArgs).stdout.toString("utf-8"); + const result = proc.spawnSync("git", logArgs); + if (result.error) { + console.log("Cannot access git", result.error); + process.exit(1); + } + + const output = result.stdout.toString("utf-8"); const containsSkipCommit = output.indexOf(skipChangelogInfix) >= 0; return containsSkipCommit; } From c71d811dd0ff63b4c1ba09026be65e0a9c9de588 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 30 Apr 2021 22:58:54 +0200 Subject: [PATCH 29/30] CR --- .github/settings.yml | 1 + .github/workflows/docs.yml | 1 + .github/workflows/nightly.yml | 14 ++++------ RELEASES.md | 2 +- docs/distribution/nightly.md | 10 +++---- .../engine/EngineReleaseProvider.scala | 4 +-- .../runtimeversionmanager/runner/Runner.scala | 6 ++--- tools/ci/check-changelog.js | 27 ++++++++++++------- tools/ci/nightly/extract-release-notes.js | 8 ++---- 9 files changed, 37 insertions(+), 36 deletions(-) diff --git a/.github/settings.yml b/.github/settings.yml index a1cc85990c89..e148dddb4fd0 100644 --- a/.github/settings.yml +++ b/.github/settings.yml @@ -215,6 +215,7 @@ branches: - "Build and Test (ubuntu-18.04)" - "Build and Test (windows-latest)" - "Docs Check" + - "Changelog Check" - "Rust Check" - "Rust Lint" - "Rust Test Native (macOS-latest)" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 84d069468224..e100ead5af79 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -27,6 +27,7 @@ jobs: run: npm install - name: Check Docs run: npx prettier --check . + changelog-check: name: Changelog Check runs-on: ubuntu-latest diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ef9d1ebbd003..53696cdb7b3b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -6,7 +6,7 @@ on: branches: - main schedule: - - cron: "0 4 * * 2-6" # 4am from Tuesday to Saturday (i.e. after every workday) + - cron: "0 4 * * 2-6" # 4am (UTC) from Tuesday to Saturday (i.e. after every workday) env: # Please ensure that this is in sync with graalVersion in build.sbt @@ -22,7 +22,7 @@ concurrency: "releases" jobs: preflight-check: - name: Check If The Build Should Proceed + name: Nightly Preflight Check runs-on: ubuntu-18.04 timeout-minutes: 10 if: @@ -34,12 +34,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 - - name: Debug Print - shell: bash - run: | - echo ${{ github.event.head_commit.message }} - echo ${{ github.event_name }} - echo ${{ github.event_name == 'schedule' || contains(github.event.head_commit.message,'[release: nightly]') }} - id: preparations name: Check If The Build Should Proceed And Prepare Version String shell: bash @@ -52,7 +46,9 @@ jobs: npm install node preflight-check.js ${{ github.sha }} - # This job should be kept up-to-date with scala.yml#build (but keep the relevant changes) + # This job should be kept up-to-date with release.yml#build (but keep the relevant changes) + # The difference is the version bump which modifies the version in build.sbt to the output of the preflight check. + # It should be done before any actual SBT builds steps. build: name: Build runs-on: ${{ matrix.os }} diff --git a/RELEASES.md b/RELEASES.md index 28ffdf79dca7..0d47c802254f 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -2,7 +2,7 @@ ## Miscellaneous -- Adding a pipeline for automatic nightly builds +- Adding a pipeline for automatic nightly builds - during the night after each workday any new changes to the `main` branch are built and released as a nightly builds. The nightly builds can be useful to preview in-development features, but they should not be relied on, as they are not considered stable and only the 3 latest nightly builds are kept, so the nightly versions become obsolete very fast. ([#1689](https://github.com/enso-org/enso/pull/1689)). # Enso 0.2.11 (2021-04-28) diff --git a/docs/distribution/nightly.md b/docs/distribution/nightly.md index ef62d5b2ed02..6feabc740cdf 100644 --- a/docs/distribution/nightly.md +++ b/docs/distribution/nightly.md @@ -8,7 +8,7 @@ order: 10 # Nightly Builds -This document describes the infrastructure for automated nightly builds. +This document describes the infrastructure for Enso's automated nightly builds. @@ -20,7 +20,7 @@ This document describes the infrastructure for automated nightly builds. ## Triggering the Build -The build may be triggered on two possible events: +The build can be triggered by two possible events: - automatically, at 4am UTC after each working day (that is, on Tuesday to Saturday), @@ -33,12 +33,12 @@ when it was triggered. However, when a nightly build is triggered (by any of the two above conditions), it will only proceed if there are any changes. That is, if the current commit is the same as the one used for the previous nightly build, the build will not -proceed, because there are no changes. +proceed because there are no changes. Thanks to [GitHub's concurrency settings](https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency), only one nightly or regular release job may be running at a given time. If a -build is triggered when another one is in progress, it will be pending until the +build is triggered while another one is in progress, it will be pending until the first one finishes. ## Nightly Build Versions @@ -58,7 +58,7 @@ subsequent build based off of the same version is triggered on the same day it will be `1.2.3-SNAPSHOT.2021-02-01.1` etc. Only the 3 most recent nightly builds are kept in the repository, any older -builds are removed from releases. +builds are removed from the releases page and their corresponding tags are also removed. ## Release Notes diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/engine/EngineReleaseProvider.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/engine/EngineReleaseProvider.scala index 346f2383cfd8..60f777a60c72 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/engine/EngineReleaseProvider.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/engine/EngineReleaseProvider.scala @@ -58,8 +58,8 @@ class EngineReleaseProvider(releaseProvider: SimpleReleaseProvider) */ private def wrapFetchError[A](result: Try[A]): Try[A] = result.recoverWith { case ReleaseNotFound(tag, _, cause) if tag.contains(Engine.nightlyInfix) => - // TODO [RW] shouldn't we somehow explain *how* to upgrade? Ideally the - // launcher should include a command for that. + // TODO [RW] explain how to upgrade using the upgrade command once its + // implemented (#1717) Failure( ReleaseNotFound( tag, diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/runner/Runner.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/runner/Runner.scala index c3f59e195a99..215915dbe8bb 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/runner/Runner.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/runner/Runner.scala @@ -57,9 +57,9 @@ class Runner( // TODO [RW] reporting warnings to the IDE (#1710) if (Engine.isNightly(engineVersion)) { Logger[Runner].warn( - "Creating a new project using a nightly build. Nightly builds may " + - "disappear after a while, so you may need to upgrade. Consider " + - "using a stable version." + s"Creating a new project using a nightly build ($engineVersion). " + + "Nightly builds may disappear after a while, so you may need to " + + "upgrade. Consider using a stable version." ) } RunSettings(engineVersion, arguments, connectLoggerIfAvailable = false) diff --git a/tools/ci/check-changelog.js b/tools/ci/check-changelog.js index 1c5fcad56798..3bdfe1f3bbe1 100644 --- a/tools/ci/check-changelog.js +++ b/tools/ci/check-changelog.js @@ -5,6 +5,19 @@ const skipChangelogInfix = "[no-changelog]"; const changelogPath = process.argv[2]; const baseRef = process.argv[3]; +/// Runs the git command with the provided arguments. +function runGit(args) { + const result = proc.spawnSync("git", args); + if (result.error) { + console.log("Cannot access git", result.error); + process.exit(1); + } + return result; +} + +/** Checks if the changelog file was changed in any commits that are part of the + * PR. + */ function wasChangelogModified() { const diffArgs = [ "--no-pager", @@ -14,24 +27,18 @@ function wasChangelogModified() { "--", changelogPath, ]; - const result = proc.spawnSync("git", diffArgs); - if (result.error) { - console.log("Cannot access git", result.error); - process.exit(1); - } + + const result = runGit(diffArgs); const exitCode = result.status; console.log(result.stdout.toString("utf-8")); const noDifference = exitCode == 0; return !noDifference; } +/// Checks if any commit has overridden the changelog check. function isChangelogSkipped() { const logArgs = ["--no-pager", "log", "HEAD~3...HEAD", "--pretty=oneline"]; - const result = proc.spawnSync("git", logArgs); - if (result.error) { - console.log("Cannot access git", result.error); - process.exit(1); - } + const result = runGit(logArgs); const output = result.stdout.toString("utf-8"); const containsSkipCommit = output.indexOf(skipChangelogInfix) >= 0; diff --git a/tools/ci/nightly/extract-release-notes.js b/tools/ci/nightly/extract-release-notes.js index 22bff26a3c49..e0b94c7869c6 100644 --- a/tools/ci/nightly/extract-release-notes.js +++ b/tools/ci/nightly/extract-release-notes.js @@ -2,7 +2,6 @@ const fs = require("fs"); const inputPath = process.argv[2]; const outputPath = process.argv[3]; -const version = process.env.DIST_VERSION; console.log("Extracting release notes from " + inputPath + " to " + outputPath); @@ -14,7 +13,7 @@ function cutFirstSection(content) { function findNightlySectionStart(text) { return text.search(nightlySectionRegex); } - const regularSectionRegex = /^# Enso .*? \(\d+-\d+-\d+\)$/gm; + const regularSectionRegex = /^# Enso .*? \(\d\d\d\d-\d\d-\d\d\)$/gm; function findFirstRegularSectionStart(text) { return text.search(regularSectionRegex); } @@ -44,10 +43,7 @@ function cutFirstSection(content) { } const firstSectionContent = rest.substring(0, secondHeading); - - const firstSectionHeader = "# Enso Nightly " + version + "\n"; - - return firstSectionHeader + firstSectionContent; + return firstSectionContent; } try { From 0468dc5bbbbf7de0d7dafda466d868128f5c846f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 30 Apr 2021 23:02:53 +0200 Subject: [PATCH 30/30] Formatting --- RELEASES.md | 8 ++++++-- docs/distribution/nightly.md | 7 ++++--- tools/ci/check-changelog.js | 4 ++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index 0d47c802254f..05b167869616 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -2,8 +2,12 @@ ## Miscellaneous -- Adding a pipeline for automatic nightly builds - during the night after each workday any new changes to the `main` branch are built and released as a nightly builds. The nightly builds can be useful to preview in-development features, but they should not be relied on, as they are not considered stable and only the 3 latest nightly builds are kept, so the nightly versions become obsolete very fast. - ([#1689](https://github.com/enso-org/enso/pull/1689)). +- Adding a pipeline for automatic nightly builds - during the night after each + workday any new changes to the `main` branch are built and released as a + nightly builds. The nightly builds can be useful to preview in-development + features, but they should not be relied on, as they are not considered stable + and only the 3 latest nightly builds are kept, so the nightly versions become + obsolete very fast. ([#1689](https://github.com/enso-org/enso/pull/1689)). # Enso 0.2.11 (2021-04-28) diff --git a/docs/distribution/nightly.md b/docs/distribution/nightly.md index 6feabc740cdf..8e014efec218 100644 --- a/docs/distribution/nightly.md +++ b/docs/distribution/nightly.md @@ -38,8 +38,8 @@ proceed because there are no changes. Thanks to [GitHub's concurrency settings](https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency), only one nightly or regular release job may be running at a given time. If a -build is triggered while another one is in progress, it will be pending until the -first one finishes. +build is triggered while another one is in progress, it will be pending until +the first one finishes. ## Nightly Build Versions @@ -58,7 +58,8 @@ subsequent build based off of the same version is triggered on the same day it will be `1.2.3-SNAPSHOT.2021-02-01.1` etc. Only the 3 most recent nightly builds are kept in the repository, any older -builds are removed from the releases page and their corresponding tags are also removed. +builds are removed from the releases page and their corresponding tags are also +removed. ## Release Notes diff --git a/tools/ci/check-changelog.js b/tools/ci/check-changelog.js index 3bdfe1f3bbe1..c3f1f3a5f0a7 100644 --- a/tools/ci/check-changelog.js +++ b/tools/ci/check-changelog.js @@ -16,8 +16,8 @@ function runGit(args) { } /** Checks if the changelog file was changed in any commits that are part of the - * PR. - */ + * PR. + */ function wasChangelogModified() { const diffArgs = [ "--no-pager",