diff --git a/.github/workflows/detect-changes-matrix.yml b/.github/workflows/detect-changes-matrix.yml index 91dd9c31f..4973fe16c 100644 --- a/.github/workflows/detect-changes-matrix.yml +++ b/.github/workflows/detect-changes-matrix.yml @@ -11,6 +11,9 @@ on: core: value: ${{ jobs.detect-changes.outputs.core }} description: "Determine if any core changes per git commit changes" + infra: + value: ${{ jobs.detect-changes.outputs.infra }} + description: "Determine if any changes to docker infra" jobs: detect-changes: @@ -20,6 +23,7 @@ jobs: matrix: ${{ steps.set-all-matrix.outputs.matrix }} integrations: ${{ steps.set-all-matrix.outputs.integrations }} core: ${{ steps.set-all-matrix.outputs.core }} + infra: ${{ steps.set-all-matrix.outputs.infra}} steps: - name: Checkout Repo uses: actions/checkout@v4 @@ -42,6 +46,8 @@ jobs: - 'integrations/**' - '!integrations/**/*.md' - '!integrations/_infra/*' + infra: + - 'integrations/_infra/*' - name: Set integrations and all matrix id: set-all-matrix @@ -50,8 +56,11 @@ jobs: HAS_CORE=${{ steps.changed-files.outputs.core_all_changed_files != '[]' }} echo "Core changes : ${HAS_CORE}" MATRIX=$(node -e "integrations=${INTEGRATIONS}; hasCore=${HAS_CORE}; console.log(JSON.stringify(hasCore ? integrations.concat(['.']) : integrations))") + HAS_INFRA=${{ steps.changed-files.outputs.infra_all_changed_files != '[]' }} + echo "Infra changes : ${HAS_INFRA}" echo "Integration changes : ${INTEGRATIONS}" echo "All changes : ${MATRIX}" echo "core=${HAS_CORE}" >> $GITHUB_OUTPUT echo "integrations=${INTEGRATIONS}" >> $GITHUB_OUTPUT echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT + echo "infra=${HAS_INFRA}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/infra.yml b/.github/workflows/infra.yml new file mode 100644 index 000000000..201c8251b --- /dev/null +++ b/.github/workflows/infra.yml @@ -0,0 +1,51 @@ +name: Build infra images +on: + pull_request: + workflow_dispatch: + +jobs: + detect-changes: + uses: ./.github/workflows/detect-changes-matrix.yml + build-infra: + # runs-on: ${{ matrix.platform == 'linux/arm64' && 'macos-13' || 'ubuntu-latest' }} + runs-on: 'ubuntu-latest' + needs: detect-changes + if: ${{ needs.detect-changes.outputs.infra == 'true' }} + steps: + - name: Check out code + uses: actions/checkout@v4 + + # - name: Setup docker (missing on MacOS) + # if: matrix.platform == 'linux/arm64' + # uses: douglascamata/setup-docker-macos-action@v1-alpha + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ secrets.DOCKER_MACHINE_USER }} + password: ${{ secrets.DOCKER_MACHINE_TOKEN }} + + - name: Build Builder Image + uses: docker/build-push-action@v6 + with: + context: . + file: ./integrations/_infra/Dockerfile.base.builder + platforms: linux/amd64,linux/arm64 + push: true + tags: ghcr.io/port-labs/port-ocean-base-builder:latest + + - name: Build Runner Image + uses: docker/build-push-action@v6 + with: + context: . + file: ./integrations/_infra/Dockerfile.base.runner + platforms: linux/amd64,linux/arm64 + push: true + tags: ghcr.io/port-labs/port-ocean-base-runner:latest diff --git "a/\\" "b/\\" new file mode 100644 index 000000000..3b89a6679 --- /dev/null +++ "b/\\" @@ -0,0 +1,59 @@ +name: Build infra images +on: + pull_request: + workflow_dispatch: + +jobs: + detect-changes: + uses: ./.github/workflows/detect-changes-matrix.yml + debug-the-things: + needs: detect-changes + runs-on: 'ubuntu-latest' + steps: + - run: | + echo "${{ needs.detect-changes.outputs.infra == 'true' }}" + echo "${{ needs.detect-changes.outputs.infra }}" + echo "${{ toJson(needs.detect-changes.outputs) }}" + build-infra: + # runs-on: ${{ matrix.platform == 'linux/arm64' && 'macos-13' || 'ubuntu-latest' }} + runs-on: 'ubuntu-latest' + needs: detect-changes + if: ${{ needs.detect-changes.outputs.infra == 'true' }} + steps: + - name: Check out code + uses: actions/checkout@v4 + + # - name: Setup docker (missing on MacOS) + # if: matrix.platform == 'linux/arm64' + # uses: douglascamata/setup-docker-macos-action@v1-alpha + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ secrets.DOCKER_MACHINE_USER }} + password: ${{ secrets.DOCKER_MACHINE_TOKEN }} + + - name: Build Builder Image + uses: docker/build-push-action@v6 + with: + context: . + file: ./integrations/_infra/Dockerfile.base.builder + platforms: linux/amd64,linux/arm64 + push: true + tags: ghcr.io/port-labs/port-ocean-base-builder:latest + + - name: Build Runner Image + uses: docker/build-push-action@v6 + with: + context: . + file: ./integrations/_infra/Dockerfile.base.runner + platforms: linux/amd64,linux/arm64 + push: true + tags: ghcr.io/port-labs/port-ocean-base-runner:latest diff --git a/integrations/_infra/Dockerfile.Deb b/integrations/_infra/Dockerfile.Deb index eebce4cda..5cd9cfba0 100644 --- a/integrations/_infra/Dockerfile.Deb +++ b/integrations/_infra/Dockerfile.Deb @@ -1,6 +1,7 @@ -ARG BASE_PYTHON_IMAGE=debian:trixie-slim +ARG BASE_BUILDER_PYTHON_IMAGE=ghcr.io/port-labs/port-ocean-base-builder:latest +ARG BASE_RUNNER_PYTHON_IMAGE=ghcr.io/port-labs/port-ocean-base-runner:latest -FROM ${BASE_PYTHON_IMAGE} AS base +FROM ${BASE_BUILDER_PYTHON_IMAGE} AS base ARG BUILD_CONTEXT ARG BUILDPLATFORM @@ -10,29 +11,13 @@ ENV LIBRDKAFKA_VERSION=1.9.2 \ POETRY_VIRTUALENVS_IN_PROJECT=1 \ PIP_ROOT_USER_ACTION=ignore -RUN apt-get update \ - && apt-get install -y \ - --no-install-recommends \ - wget \ - g++ \ - libssl-dev \ - autoconf \ - automake \ - libtool \ - curl \ - librdkafka-dev \ - python3 \ - python3-pip \ - python3-poetry \ - && apt-get clean - WORKDIR /app COPY ./${BUILD_CONTEXT}/pyproject.toml ./${BUILD_CONTEXT}/poetry.lock /app/ RUN poetry install --without dev --no-root --no-interaction --no-ansi --no-cache -FROM ${BASE_PYTHON_IMAGE} AS prod +FROM ${BASE_RUNNER_PYTHON_IMAGE} AS prod ARG INTEGRATION_VERSION ARG BUILD_CONTEXT @@ -45,10 +30,6 @@ LABEL org.opencontainers.image.source=https://github.com/port-labs/ocean ENV PIP_ROOT_USER_ACTION=ignore -RUN apt-get update \ - && apt-get install -y --no-install-recommends librdkafka-dev python3 \ - && apt-get clean - WORKDIR /app # Copy the application code diff --git a/integrations/_infra/Dockerfile.base.builder b/integrations/_infra/Dockerfile.base.builder new file mode 100644 index 000000000..4121f52e3 --- /dev/null +++ b/integrations/_infra/Dockerfile.base.builder @@ -0,0 +1,25 @@ +ARG BASE_PYTHON_IMAGE=debian:trixie-slim +FROM ${BASE_PYTHON_IMAGE} + +LABEL org.opencontainers.image.source=https://github.com/port-labs/ocean + +ENV LIBRDKAFKA_VERSION=1.9.2 \ + PYTHONUNBUFFERED=1 \ + POETRY_VIRTUALENVS_IN_PROJECT=1 \ + PIP_ROOT_USER_ACTION=ignore + +RUN apt-get update \ + && apt-get install -y \ + --no-install-recommends \ + wget \ + g++ \ + libssl-dev \ + autoconf \ + automake \ + libtool \ + curl \ + librdkafka-dev \ + python3 \ + python3-pip \ + python3-poetry \ + && apt-get clean diff --git a/integrations/_infra/Dockerfile.base.runner b/integrations/_infra/Dockerfile.base.runner new file mode 100644 index 000000000..48061584d --- /dev/null +++ b/integrations/_infra/Dockerfile.base.runner @@ -0,0 +1,12 @@ +ARG BASE_PYTHON_IMAGE=debian:trixie-slim +FROM ${BASE_PYTHON_IMAGE} + +LABEL org.opencontainers.image.source=https://github.com/port-labs/ocean + +ENV LIBRDKAFKA_VERSION=1.9.2 + +ENV PIP_ROOT_USER_ACTION=ignore + +RUN apt-get update \ + && apt-get install -y --no-install-recommends librdkafka-dev python3 \ + && apt-get clean