Separate builder and runner image

.github/workflows/detect-changes-matrix.yml

@@ -11,6 +11,9 @@ on:
       core:
         value: ${{ jobs.detect-changes.outputs.core }}
         description: "Determine if any core changes per git commit changes"
+      infra:
+        value: ${{ jobs.detect-changes.outputs.infra }}
+        description: "Determine if any changes to docker infra"
 
 jobs:
   detect-changes:
@@ -20,6 +23,7 @@ jobs:
       matrix: ${{ steps.set-all-matrix.outputs.matrix }}
       integrations: ${{ steps.set-all-matrix.outputs.integrations }}
       core: ${{ steps.set-all-matrix.outputs.core }}
+      infra: ${{ steps.set-all-matrix.outputs.infra}}
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v4
@@ -42,6 +46,8 @@ jobs:
               - 'integrations/**'
               - '!integrations/**/*.md'
               - '!integrations/_infra/*'
+            infra:
+              - 'integrations/_infra/*'
 
       - name: Set integrations and all matrix
         id: set-all-matrix
@@ -50,8 +56,11 @@ jobs:
           HAS_CORE=${{ steps.changed-files.outputs.core_all_changed_files != '[]' }}
           echo "Core changes : ${HAS_CORE}"
           MATRIX=$(node -e "integrations=${INTEGRATIONS}; hasCore=${HAS_CORE}; console.log(JSON.stringify(hasCore ? integrations.concat(['.']) : integrations))")
+          HAS_INFRA=${{ steps.changed-files.outputs.infra_all_changed_files != '[]' }}
+          echo "Infra changes : ${HAS_INFRA}"
           echo "Integration changes : ${INTEGRATIONS}"
           echo "All changes : ${MATRIX}"
           echo "core=${HAS_CORE}" >> $GITHUB_OUTPUT
           echo "integrations=${INTEGRATIONS}" >> $GITHUB_OUTPUT
           echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT
+          echo "infra=${HAS_INFRA}" >> $GITHUB_OUTPUT

.github/workflows/infra.yml

@@ -0,0 +1,51 @@
+name: Build infra images
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  detect-changes:
+    uses: ./.github/workflows/detect-changes-matrix.yml
+  build-infra:
+    # runs-on: ${{ matrix.platform == 'linux/arm64' && 'macos-13' || 'ubuntu-latest' }}
+    runs-on: 'ubuntu-latest'
+    needs: detect-changes
+    if: ${{ needs.detect-changes.outputs.infra == 'true' }}
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      # - name: Setup docker (missing on MacOS)
+      #   if: matrix.platform == 'linux/arm64'
+      #   uses: douglascamata/setup-docker-macos-action@v1-alpha
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ secrets.DOCKER_MACHINE_USER }}
+          password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
+
+      - name: Build Builder Image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./integrations/_infra/Dockerfile.base.builder
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ghcr.io/port-labs/port-ocean-base-builder:latest
+
+      - name: Build Runner Image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./integrations/_infra/Dockerfile.base.runner
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ghcr.io/port-labs/port-ocean-base-runner:latest

\

@@ -0,0 +1,59 @@
+name: Build infra images
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  detect-changes:
+    uses: ./.github/workflows/detect-changes-matrix.yml
+  debug-the-things:
+    needs: detect-changes
+    runs-on: 'ubuntu-latest'
+    steps:
+      - run: |
+          echo "${{ needs.detect-changes.outputs.infra == 'true' }}"
+          echo "${{ needs.detect-changes.outputs.infra }}"
+          echo "${{ toJson(needs.detect-changes.outputs) }}"
+  build-infra:
+    # runs-on: ${{ matrix.platform == 'linux/arm64' && 'macos-13' || 'ubuntu-latest' }}
+    runs-on: 'ubuntu-latest'
+    needs: detect-changes
+    if: ${{ needs.detect-changes.outputs.infra == 'true' }}
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      # - name: Setup docker (missing on MacOS)
+      #   if: matrix.platform == 'linux/arm64'
+      #   uses: douglascamata/setup-docker-macos-action@v1-alpha
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ secrets.DOCKER_MACHINE_USER }}
+          password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
+
+      - name: Build Builder Image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./integrations/_infra/Dockerfile.base.builder
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ghcr.io/port-labs/port-ocean-base-builder:latest
+
+      - name: Build Runner Image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./integrations/_infra/Dockerfile.base.runner
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ghcr.io/port-labs/port-ocean-base-runner:latest

integrations/_infra/Dockerfile.Deb

@@ -1,6 +1,7 @@
-ARG BASE_PYTHON_IMAGE=debian:trixie-slim
+ARG BASE_BUILDER_PYTHON_IMAGE=ghcr.io/port-labs/port-ocean-base-builder:latest
+ARG BASE_RUNNER_PYTHON_IMAGE=ghcr.io/port-labs/port-ocean-base-runner:latest
 
-FROM ${BASE_PYTHON_IMAGE} AS base
+FROM ${BASE_BUILDER_PYTHON_IMAGE} AS base
 
 ARG BUILD_CONTEXT
 ARG BUILDPLATFORM
@@ -10,29 +11,13 @@ ENV LIBRDKAFKA_VERSION=1.9.2 \
     POETRY_VIRTUALENVS_IN_PROJECT=1 \
     PIP_ROOT_USER_ACTION=ignore
 
-RUN apt-get update \
-    && apt-get install -y \
-        --no-install-recommends \
-        wget \
-        g++ \
-        libssl-dev \
-        autoconf \
-        automake \
-        libtool \
-        curl \
-        librdkafka-dev \
-        python3 \
-        python3-pip \
-        python3-poetry \
-    && apt-get clean
-
 WORKDIR /app
 
 COPY ./${BUILD_CONTEXT}/pyproject.toml ./${BUILD_CONTEXT}/poetry.lock /app/
 
 RUN poetry install --without dev --no-root --no-interaction --no-ansi --no-cache
 
-FROM ${BASE_PYTHON_IMAGE} AS prod
+FROM ${BASE_RUNNER_PYTHON_IMAGE} AS prod
 
 ARG INTEGRATION_VERSION
 ARG BUILD_CONTEXT
@@ -45,10 +30,6 @@ LABEL org.opencontainers.image.source=https://github.com/port-labs/ocean
 
 ENV PIP_ROOT_USER_ACTION=ignore
 
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends librdkafka-dev python3 \
-    && apt-get clean
-
 WORKDIR /app
 
 # Copy the application code

integrations/_infra/Dockerfile.base.builder

@@ -0,0 +1,25 @@
+ARG BASE_PYTHON_IMAGE=debian:trixie-slim
+FROM ${BASE_PYTHON_IMAGE}
+
+LABEL org.opencontainers.image.source=https://github.com/port-labs/ocean
+
+ENV LIBRDKAFKA_VERSION=1.9.2 \
+    PYTHONUNBUFFERED=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    PIP_ROOT_USER_ACTION=ignore
+
+RUN apt-get update \
+    && apt-get install -y \
+        --no-install-recommends \
+        wget \
+        g++ \
+        libssl-dev \
+        autoconf \
+        automake \
+        libtool \
+        curl \
+        librdkafka-dev \
+        python3 \
+        python3-pip \
+        python3-poetry \
+    && apt-get clean

integrations/_infra/Dockerfile.base.runner

@@ -0,0 +1,12 @@
+ARG BASE_PYTHON_IMAGE=debian:trixie-slim
+FROM ${BASE_PYTHON_IMAGE}
+
+LABEL org.opencontainers.image.source=https://github.com/port-labs/ocean
+
+ENV LIBRDKAFKA_VERSION=1.9.2
+
+ENV PIP_ROOT_USER_ACTION=ignore
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends librdkafka-dev python3 \
+    && apt-get clean