From 21a44bcb559682f67289ac72571206597c00855b Mon Sep 17 00:00:00 2001 From: Ruffin Date: Mon, 19 Jul 2021 15:49:37 -0700 Subject: [PATCH] Refactor actions to rebuild CI image from CI (#2405) * Add dependabot config * Workarround missing ARG support in dependabot Parrent image can't yet be parameterized for version parsing https://github.com/dependabot/dependabot-core/issues/2057 * Fix CCACHE_DIR to be absolute path https://github.com/ros-planning/navigation2/pull/2403 * Remove FAIL_ON_BUILD_FAILURE ARG now that CI image builds from rolling testing and no longer ros2 nightly so forcing image rebuilds is no longer needed * Trigger rebuild on changes to Dockerfile * Add github-actions to dependabot * Refactor actions to rebuild CI image from CI * Clean old dockerhub build hook * Rename action * Combine workflows to avoid dispaching as that requires a github personal access token which is an inconvenience to generate and add to repo secrets * Add needs param to define job order * Use job outputs for condition instead of env Seems that env in context is not yet avalable at the job level but only at the current job's step level https://github.community/t/how-to-set-and-access-a-workflow-variable/17335/6 https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idoutputs * Use multi-line syntax for tags param * Use toJSON for job conditional * Test with always * Update param syntax * Test with always * Update if syntax * Test if trigger for only check_ci_files * Simplify if params * Try using GitHub's Container registry * Indent formating * Add step conditional * Default arg to build from unpinned parrent stage allowing CI to build from pinned testing image while enabling dependabot to find and update testing image version ensuring CI images are up-to-date with upstream registries * Set build arg for continuous integration * Ignore .github folder * testing * Update colcon-cache * Revert to rolling * Revert testing * Update README --- .circleci/config.yml | 8 +- .dockerhub/builder/Dockerfile | 1 - .dockerhub/builder/hooks/build | 14 --- .dockerignore | 2 +- .github/dependabot.yml | 14 +++ .github/workflows/trigger_dockerhub.yaml | 52 ----------- .github/workflows/update_ci_image.yaml | 107 +++++++++++++++++++++++ Dockerfile | 14 ++- README.md | 4 +- {.dockerhub => tools}/distro.Dockerfile | 0 {.dockerhub => tools}/source.Dockerfile | 0 11 files changed, 135 insertions(+), 81 deletions(-) delete mode 120000 .dockerhub/builder/Dockerfile delete mode 100755 .dockerhub/builder/hooks/build create mode 100644 .github/dependabot.yml delete mode 100644 .github/workflows/trigger_dockerhub.yaml create mode 100644 .github/workflows/update_ci_image.yaml rename {.dockerhub => tools}/distro.Dockerfile (100%) rename {.dockerhub => tools}/source.Dockerfile (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1cd0a121f8..ac878e3712 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -263,7 +263,7 @@ _commands: colcon clean packages --yes \ --packages-select ${TEST_PACKAGES} \ - --base-select test_result + --base-select test_result . install/setup.sh set -o xtrace @@ -302,7 +302,7 @@ _steps: echo $CACHE_NONCE | \ (echo cache_nonce && cat) >> lockfile.txt sha256sum $PWD/lockfile.txt >> lockfile.txt - + TZ=utc stat -c '%y' /ros_entrypoint.sh | \ (echo ros_entrypoint && cat) >> lockfile.txt sha256sum $PWD/lockfile.txt >> lockfile.txt @@ -436,7 +436,7 @@ _environments: executors: debug_exec: docker: - - image: rosplanning/navigation2:main + - image: ghcr.io/ros-planning/navigation2:main working_directory: /opt/overlay_ws environment: <<: *common_environment @@ -444,7 +444,7 @@ executors: OVERLAY_MIXINS: "debug ccache coverage-gcc" release_exec: docker: - - image: rosplanning/navigation2:main + - image: ghcr.io/ros-planning/navigation2:main working_directory: /opt/overlay_ws environment: <<: *common_environment diff --git a/.dockerhub/builder/Dockerfile b/.dockerhub/builder/Dockerfile deleted file mode 120000 index 36c49d2a30..0000000000 --- a/.dockerhub/builder/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -../../Dockerfile \ No newline at end of file diff --git a/.dockerhub/builder/hooks/build b/.dockerhub/builder/hooks/build deleted file mode 100755 index 8687b011e9..0000000000 --- a/.dockerhub/builder/hooks/build +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -set -ex - -export FROM_IMAGE=osrf/ros2:testing -export FAIL_ON_BUILD_FAILURE="" -export UNDERLAY_MIXINS="release ccache" - -docker build \ - --target builder \ - --tag ${IMAGE_NAME} \ - --build-arg FROM_IMAGE \ - --build-arg FAIL_ON_BUILD_FAILURE \ - --build-arg UNDERLAY_MIXINS \ - --file ../../Dockerfile ../../. diff --git a/.dockerignore b/.dockerignore index 7bf7802aec..5e31fef9d5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,7 +2,7 @@ # Repo .circleci/ -.dockerhub/ +.github/ .git/ .github/ .dockerignore diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..3017359824 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,14 @@ +version: 2 +updates: + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "daily" + commit-message: + prefix: "🐳" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + commit-message: + prefix: "🛠️" diff --git a/.github/workflows/trigger_dockerhub.yaml b/.github/workflows/trigger_dockerhub.yaml deleted file mode 100644 index f4ba5d032c..0000000000 --- a/.github/workflows/trigger_dockerhub.yaml +++ /dev/null @@ -1,52 +0,0 @@ ---- -name: Trigger Docker Hub - -on: - schedule: - # 7am UTC, 12am PDT - - cron: '0 7 * * *' - push: - branches: - - main - paths: - - '**/package.xml' - - '**/*.repos' - -jobs: - rebuild_dockerhub: - name: Trigger Docker Hub - runs-on: ubuntu-latest - container: - image: rosplanning/navigation2:main - steps: - - name: "Initialize environment" - run: | - echo "TRIGGER=false" >> $GITHUB_ENV - - name: "Check apt updates" - if: ${{ github.event_name == 'schedule' }} - env: - SOURCELIST: sources.list.d/ros2.list - run: | - apt-get update \ - -o Dir::Etc::sourcelist="${SOURCELIST}" - apt-get --simulate upgrade \ - -o Dir::Etc::sourcelist="${SOURCELIST}" \ - | grep "0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." \ - && echo "No apt updates" || echo "TRIGGER=true" >> $GITHUB_ENV - - name: "Check package updates" - if: ${{ github.event_name == 'push' }} - run: | - echo "TRIGGER=true" >> $GITHUB_ENV - - name: "Trigger Dockerhub URL" - if: ${{ fromJSON(env.TRIGGER) }} - env: - DATA: | - { - "source_type": "Branch", - "source_name": "main" - } - run: | - echo ${DATA} \ - | curl -H "Content-Type: application/json" \ - --data @- \ - -X POST ${{ secrets.DOCKERHUB_TRIGGER_URL }} diff --git a/.github/workflows/update_ci_image.yaml b/.github/workflows/update_ci_image.yaml new file mode 100644 index 0000000000..6ed2f64225 --- /dev/null +++ b/.github/workflows/update_ci_image.yaml @@ -0,0 +1,107 @@ +--- +name: Update CI Image + +on: + schedule: + # 7am UTC, 12am PDT + - cron: '0 7 * * *' + push: + branches: + - main + paths: + - '**/package.xml' + - '**/*.repos' + - 'Dockerfile' + +jobs: + check_ci_files: + name: Check CI Files + runs-on: ubuntu-latest + outputs: + trigger: ${{ steps.check.outputs.trigger }} + no_cache: ${{ steps.check.outputs.no_cache }} + steps: + - name: "Check package updates" + id: check + if: github.event_name == 'push' + run: | + echo "::set-output name=trigger::true" + echo "::set-output name=no_cache::false" + check_ci_image: + name: Check CI Image + if: github.event_name == 'schedule' + needs: check_ci_files + runs-on: ubuntu-latest + outputs: + trigger: ${{ steps.check.outputs.trigger }} + no_cache: ${{ steps.check.outputs.no_cache }} + container: + image: ghcr.io/ros-planning/navigation2:main + steps: + - name: "Check apt updates" + id: check + env: + SOURCELIST: sources.list.d/ros2.list + run: | + apt-get update \ + -o Dir::Etc::sourcelist="${SOURCELIST}" + apt-get --simulate upgrade \ + -o Dir::Etc::sourcelist="${SOURCELIST}" \ + | grep "0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." \ + && echo "::set-output name=trigger::false" \ + || echo "::set-output name=trigger::true" + echo "::set-output name=no_cache::true" + rebuild_ci_image: + name: Rebuild CI Image + if: always() + needs: + - check_ci_files + - check_ci_image + runs-on: ubuntu-latest + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set build config + id: config + run: | + timestamp=$(date --utc +%Y%m%d%H%M%S) + echo "::set-output name=timestamp::${timestamp}" + + no_cache=false + if [ "${{needs.check_ci_files.outputs.no_cache}}" == 'true' ] || \ + [ "${{needs.check_ci_image.outputs.no_cache}}" == 'true' ] + then + no_cache=true + fi + echo "::set-output name=no_cache::${no_cache}" + + trigger=false + if [ "${{needs.check_ci_files.outputs.trigger}}" == 'true' ] || \ + [ "${{needs.check_ci_image.outputs.trigger}}" == 'true' ] + then + trigger=true + fi + echo "::set-output name=trigger::${trigger}" + - name: Build and push + if: steps.config.outputs.trigger == 'true' + id: docker_build + uses: docker/build-push-action@v2 + with: + pull: true + push: true + no-cache: ${{ steps.config.outputs.no_cache }} + cache-from: type=registry,ref=ghcr.io/ros-planning/navigation2:main + cache-to: type=inline + target: builder + tags: | + ghcr.io/ros-planning/navigation2:main + ghcr.io/ros-planning/navigation2:main-${{ steps.config.outputs.timestamp }} + - name: Image digest + if: steps.config.outputs.trigger == 'true' + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/Dockerfile b/Dockerfile index 5ea809319f..42ec564378 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ # docker build -t nav2:latest \ # --build-arg UNDERLAY_MIXINS \ # --build-arg OVERLAY_MIXINS ./ -ARG FROM_IMAGE=osrf/ros2:testing +ARG FROM_IMAGE=ros:rolling ARG UNDERLAY_WS=/opt/underlay_ws ARG OVERLAY_WS=/opt/overlay_ws @@ -57,7 +57,7 @@ RUN apt-get update && \ ros-$ROS_DISTRO-rmw-cyclonedds-cpp \ && pip3 install \ fastcov \ - git+https://github.com/ruffsl/colcon-cache.git@c1cedadc1ac6131fe825d075526ed4ae8e1b473c \ + git+https://github.com/ruffsl/colcon-cache.git@1d6ae5745ac3e124bb46c3a636439dbc02af77dd \ git+https://github.com/ruffsl/colcon-clean.git@87dee2dd1e47c2b97ac6d8300f76e3f607d19ef6 \ && rosdep update \ && rm -rf /var/lib/apt/lists/* @@ -79,15 +79,13 @@ RUN . /opt/ros/$ROS_DISTRO/setup.sh && \ # build underlay source COPY --from=cacher $UNDERLAY_WS ./ ARG UNDERLAY_MIXINS="release ccache" -ARG FAIL_ON_BUILD_FAILURE=True -ARG CCACHE_DIR=".ccache" +ARG CCACHE_DIR="$UNDERLAY_WS/.ccache" RUN . /opt/ros/$ROS_DISTRO/setup.sh && \ colcon cache lock && \ colcon build \ --symlink-install \ --mixin $UNDERLAY_MIXINS \ - --event-handlers console_direct+ \ - || ([ -z "$FAIL_ON_BUILD_FAILURE" ] || exit 1) + --event-handlers console_direct+ # install overlay dependencies ARG OVERLAY_WS @@ -109,12 +107,12 @@ FROM builder AS tester # build overlay source COPY --from=cacher $OVERLAY_WS ./ ARG OVERLAY_MIXINS="release ccache" +ARG CCACHE_DIR="$OVERLAY_WS/.ccache" RUN . $UNDERLAY_WS/install/setup.sh && \ colcon cache lock && \ colcon build \ --symlink-install \ - --mixin $OVERLAY_MIXINS \ - || ([ -z "$FAIL_ON_BUILD_FAILURE" ] || exit 1) + --mixin $OVERLAY_MIXINS # source overlay from entrypoint RUN sed --in-place \ diff --git a/README.md b/README.md index 29336bd8f1..b0a0712812 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # Nav2 -[![Build Status](https://img.shields.io/docker/pulls/rosplanning/navigation2.svg?maxAge=2592000)](https://hub.docker.com/r/rosplanning/navigation2) [![Build Status](https://img.shields.io/docker/cloud/build/rosplanning/navigation2.svg?label=docker%20build)](https://hub.docker.com/r/rosplanning/navigation2) [![codecov](https://codecov.io/gh/ros-planning/navigation2/branch/main/graph/badge.svg)](https://codecov.io/gh/ros-planning/navigation2) +[![GitHub Workflow Status](https://github.com/ros-planning/navigation2/actions/workflows/update_ci_image.yaml/badge.svg)](https://github.com/ros-planning/navigation2/actions/workflows/update_ci_image.yaml) +[![codecov](https://codecov.io/gh/ros-planning/navigation2/branch/main/graph/badge.svg)](https://codecov.io/gh/ros-planning/navigation2)

@@ -14,6 +15,7 @@ For detailed instructions on how to: - [Configure](https://navigation.ros.org/configuration/index.html) - [Navigation Plugins](https://navigation.ros.org/plugins/index.html) - [Migration Guides](https://navigation.ros.org/migration/index.html) +- [Container Images](https://github.com/orgs/ros-planning/packages/container/package/navigation2) - [Contribute](https://navigation.ros.org/contribute/index.html) Please visit our [documentation site](https://navigation.ros.org/). [Please visit our community Slack here](https://join.slack.com/t/navigation2/shared_invite/zt-hu52lnnq-cKYjuhTY~sEMbZXL8p9tOw) (if this link does not work, please contact maintainers to reactivate). diff --git a/.dockerhub/distro.Dockerfile b/tools/distro.Dockerfile similarity index 100% rename from .dockerhub/distro.Dockerfile rename to tools/distro.Dockerfile diff --git a/.dockerhub/source.Dockerfile b/tools/source.Dockerfile similarity index 100% rename from .dockerhub/source.Dockerfile rename to tools/source.Dockerfile