Buildtools Dockerfile

buildtools/ contains a Dockerfile which can build the buildtools packages. This is done with Ubuntu 22.04 as it can run bitbake with the tools it gets from the live package feeds. Images which do not have the appropriate tooling in their package feeds will be able to copy the appropriate buildtools installer. This usage has been implemented in the following distros... - ubuntu-18.04 - leap-15.3 - leap-15.4 - almalinux-8.7 Many tweaks to the `Build images` workflow to allow for this to happen within github actions.
lgrosz · May 25, 2023 · 4a90f8b · 4a90f8b
1 parent 2d9d357
commit 4a90f8b
Show file tree

Hide file tree

Showing 6 changed files with 170 additions and 11 deletions.
diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
@@ -1,20 +1,62 @@
 name: Build images
+
 on:
   push:
     branches:
       - master
   pull_request:
-
   workflow_dispatch:
 
 jobs:
-  build-image:
+  buildtools:
+    runs-on: ubuntu-latest
+    services:
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
+        volumes:
+          - /tmp/registry:/var/lib/registry
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          # network=host driver-opt needed to push to local registry
+          driver-opts: network=host
+
+      - name: Build buildtools image
+        uses: docker/build-push-action@v4
+        with:
+          target: buildtools
+          context: buildtools
+          tags: localhost:5000/yocto_buildtools:latest
+          push: true
+
+      - name: Upload registry as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: docker-registry
+          path: /tmp/registry
+
+  images:
+    needs: buildtools
     runs-on: ubuntu-latest
     permissions:
       packages: write
     strategy:
       fail-fast: false
       matrix:
+        # TODO Buildtools extended steps should only need to be done for
+        # - almalinux-8.7
+        # - ubuntu-18.04,
+        # - leap-15.3 
+        # - leap-15.4
         base_distro: [
           almalinux-8.7,
           almalinux-9.1,
@@ -29,19 +71,46 @@ jobs:
         ]
 
     steps:
+      - name: Download registry from artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: docker-registry
+          path: /tmp/registry
+
+      # This must be done without the service container feature since the
+      # volume needs to be populated before starting the container
+      - name: Start conatiner registry
+        run: |
+          docker run --rm --detach --publish 5000:5000 \
+          --volume /tmp/registry:/var/lib/registry \
+          registry:2
+
       - name: Checkout the repo
         uses: actions/checkout@v3
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          # network=host driver-opt needed to push to local registry
+          driver-opts: network=host
+
       - name: Test
         uses: docker/build-push-action@v4
         with:
           target: test
           context: ${{ matrix.base_distro }}
+          build-contexts: |
+            yocto_buildtools=docker-image://localhost:5000/yocto_buildtools:latest
           push: false
 
       - name: Development
         uses: docker/build-push-action@v4
         with:
           target: development
           context: ${{ matrix.base_distro }}
+          build-contexts: |
+            yocto_buildtools=docker-image://localhost:5000/yocto_buildtools:latest
           push: false
diff --git a/almalinux-8.7/Dockerfile b/almalinux-8.7/Dockerfile
@@ -52,6 +52,10 @@ ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
+# Pull in build-tools
+COPY --from=yocto_buildtools /x86_64-buildtools-extended-nativesdk-standalone-4.2.sh /
+RUN /x86_64-buildtools-extended-nativesdk-standalone-4.2.sh -y
+
 # Can't run bitbake as root
 RUN useradd --create-home chef
 
@@ -62,7 +66,8 @@ WORKDIR /home/chef
 
 # [Quick build](https://docs.yoctoproject.org/brief-yoctoprojectqs/index.html#use-git-to-clone-poky)
 RUN \
-  git clone git://git.yoctoproject.org/poky \
+  . /opt/poky/4.2/environment-setup-x86_64-pokysdk-linux \
+  && git clone git://git.yoctoproject.org/poky \
   && cd poky \
   && git checkout master \
   && . ./oe-init-build-env \

diff --git a/buildtools/Dockerfile b/buildtools/Dockerfile
@@ -0,0 +1,69 @@
+FROM ubuntu:22.04 as base
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# [Required packages for build host](https://docs.yoctoproject.org/ref-manual/system-requirements.html#ubuntu-and-debian)
+RUN apt update && apt install -y \
+  build-essential \
+  chrpath \
+  cpio \
+  debianutils \
+  diffstat \
+  file \
+  gawk \
+  gcc \
+  git \
+  iputils-ping \
+  libegl1-mesa \
+  liblz4-tool \
+  libsdl1.2-dev \
+  mesa-common-dev \
+  pylint \
+  python3 \
+  python3-git \
+  python3-jinja2 \
+  python3-pexpect \
+  python3-pip \
+  python3-subunit \
+  socat \
+  texinfo \
+  unzip \
+  wget \
+  xterm \
+  xz-utils \
+  zstd \
+  && apt clean && rm -rf /var/lib/apt/lists/*
+
+# Locale setup
+RUN apt update && apt install -y \
+  locales \
+  && apt clean && rm -rf /var/lib/apt/lists/*
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \
+  && locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+# Can't run bitbake as root
+RUN useradd --create-home chef
+
+
+from base as builder
+
+USER chef
+WORKDIR /home/chef
+
+RUN \
+  git clone git://git.yoctoproject.org/poky \
+  && cd poky \
+  && git checkout master \
+  && . ./oe-init-build-env \
+  && bitbake buildtools-tarball buildtools-extended-tarball buildtools-make-tarball
+
+
+from ubuntu:22.04 as buildtools
+
+COPY --from=builder /home/chef/poky/build/tmp/deploy/sdk/x86_64-buildtools-nativesdk-standalone-4.2.sh /
+COPY --from=builder /home/chef/poky/build/tmp/deploy/sdk/x86_64-buildtools-extended-nativesdk-standalone-4.2.sh /
+COPY --from=builder /home/chef/poky/build/tmp/deploy/sdk/x86_64-buildtools-make-nativesdk-standalone-4.2.sh /
+
diff --git a/leap-15.3/Dockerfile b/leap-15.3/Dockerfile
@@ -4,11 +4,14 @@ FROM opensuse/leap:15.3 as base
 RUN zypper --non-interactive update && zypper --non-interactive install \
   Mesa-dri-devel \
   Mesa-libEGL1 \
+  bzip2 \
   chrpath \
   diffstat \
   gcc \
   gcc-c++ \
   git \
+  gzip \
+  hostname \
   libSDL-devel \
   lz4 \
   make \
@@ -34,6 +37,10 @@ RUN zypper --non-interactive update && zypper --non-interactive install \
 
 RUN pip3 install GitPython
 
+# Pull in build-tools
+COPY --from=yocto_buildtools /x86_64-buildtools-extended-nativesdk-standalone-4.2.sh /
+RUN /x86_64-buildtools-extended-nativesdk-standalone-4.2.sh -y
+
 # Can't run bitbake as root
 RUN useradd --user-group --create-home chef
 
@@ -44,7 +51,8 @@ WORKDIR /home/chef
 
 # [Quick build](https://docs.yoctoproject.org/brief-yoctoprojectqs/index.html#use-git-to-clone-poky)
 RUN \
-  git clone git://git.yoctoproject.org/poky \
+  . /opt/poky/4.2/environment-setup-x86_64-pokysdk-linux \
+  && git clone git://git.yoctoproject.org/poky \
   && cd poky \
   && git checkout master \
   && . ./oe-init-build-env \
@@ -55,7 +63,6 @@ FROM base as development
 
 RUN zypper --non-interactive update && zypper --non-interactive install \
   curl \
-  gzip \
   && zypper clean
 
 # Setup [fixuid](https://github.com/boxboat/fixuid)

diff --git a/leap-15.4/Dockerfile b/leap-15.4/Dockerfile
@@ -4,11 +4,14 @@ FROM opensuse/leap:15.4 as base
 RUN zypper --non-interactive update && zypper --non-interactive install \
   Mesa-dri-devel \
   Mesa-libEGL1 \
+  bzip2 \
   chrpath \
   diffstat \
   gcc \
   gcc-c++ \
   git \
+  gzip \
+  hostname \
   libSDL-devel \
   lz4 \
   make \
@@ -34,6 +37,10 @@ RUN zypper --non-interactive update && zypper --non-interactive install \
 
 RUN pip3 install GitPython
 
+# Pull in build-tools
+COPY --from=yocto_buildtools /x86_64-buildtools-extended-nativesdk-standalone-4.2.sh /
+RUN /x86_64-buildtools-extended-nativesdk-standalone-4.2.sh -y
+
 # Can't run bitbake as root
 RUN useradd --user-group --create-home chef
 
@@ -44,7 +51,8 @@ WORKDIR /home/chef
 
 # [Quick build](https://docs.yoctoproject.org/brief-yoctoprojectqs/index.html#use-git-to-clone-poky)
 RUN \
-  git clone git://git.yoctoproject.org/poky \
+  . /opt/poky/4.2/environment-setup-x86_64-pokysdk-linux \
+  && git clone git://git.yoctoproject.org/poky \
   && cd poky \
   && git checkout master \
   && . ./oe-init-build-env \
@@ -53,10 +61,6 @@ RUN \
 
 FROM base as development
 
-RUN zypper --non-interactive update && zypper --non-interactive install \
-  gzip \
-  && zypper clean
-
 # Setup [fixuid](https://github.com/boxboat/fixuid)
 RUN USER=chef && \
     GROUP=chef && \

diff --git a/ubuntu-18.04/Dockerfile b/ubuntu-18.04/Dockerfile
@@ -43,6 +43,10 @@ ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
+# Pull in build-tools
+COPY --from=yocto_buildtools /x86_64-buildtools-extended-nativesdk-standalone-4.2.sh /
+RUN /x86_64-buildtools-extended-nativesdk-standalone-4.2.sh -y
+
 # Can't run bitbake as root
 RUN useradd --create-home chef
 
@@ -53,7 +57,8 @@ WORKDIR /home/chef
 
 # [Quick build](https://docs.yoctoproject.org/brief-yoctoprojectqs/index.html#use-git-to-clone-poky)
 RUN \
-  git clone git://git.yoctoproject.org/poky \
+  . /opt/poky/4.2/environment-setup-x86_64-pokysdk-linux \
+  && git clone git://git.yoctoproject.org/poky \
   && cd poky \
   && git checkout master \
   && . ./oe-init-build-env \