From 08d4155b95f0100a08311a90c0b254f54d0383df Mon Sep 17 00:00:00 2001 From: fwcd Date: Thu, 29 Dec 2022 18:51:29 +0100 Subject: [PATCH 01/65] Run Docker workflow on all 'docker/'-prefixed branches --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5017eff6..05f27bbe 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - docker/** workflow_dispatch: jobs: From 956005d4cd2fd0951abe3bdf7c8fbe7b9b2df7f5 Mon Sep 17 00:00:00 2001 From: fwcd Date: Thu, 29 Dec 2022 19:05:28 +0100 Subject: [PATCH 02/65] Use 2-space indent in scripts --- .editorconfig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.editorconfig b/.editorconfig index 0d5248b5..fbe96814 100644 --- a/.editorconfig +++ b/.editorconfig @@ -12,3 +12,6 @@ indent_size = 2 [Package.resolved] indent_size = 2 + +[Scripts/*] +indent_size = 2 From 0011fc5ec9bf48fb9363d03c284292bf640c4009 Mon Sep 17 00:00:00 2001 From: fwcd Date: Thu, 29 Dec 2022 19:46:22 +0100 Subject: [PATCH 03/65] Extract arch-aware build scripts from Dockerfile --- Dockerfile | 10 ++++++---- Scripts/build-release | 19 ++++++++++++++++++ Scripts/setup-dotbuild-tree | 40 +++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 4 deletions(-) create mode 100755 Scripts/build-release create mode 100755 Scripts/setup-dotbuild-tree diff --git a/Dockerfile b/Dockerfile index 26223f5c..c79e2ce7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM swift:5.7-focal as builder +FROM --platform=$BUILDPLATFORM swift:5.7-focal AS builder # Install native dependencies COPY Scripts/install-build-dependencies-apt Scripts/ @@ -9,9 +9,10 @@ WORKDIR /opt/d2 COPY Sources Sources COPY Tests Tests COPY Package.swift Package.resolved ./ -RUN swift build -c release +COPY Scripts/build-release Scripts/ +RUN Scripts/build-release -FROM swift:5.7-focal-slim as runner +FROM swift:5.7-focal-slim AS runner # Install Curl, add-apt-repository and node package repository RUN apt-get update && apt-get install -y curl software-properties-common && rm -rf /var/lib/apt/lists/* @@ -34,7 +35,8 @@ COPY LICENSE README.md ./ # Set up .build folder in runner WORKDIR /opt/d2/.build -RUN mkdir -p x86_64-unknown-linux-gnu/release && ln -s x86_64-unknown-linux-gnu/release release +COPY Scripts/setup-dotbuild-tree Scripts/ +RUN Scripts/setup-dotbuild-tree # Copy font used by swiftplot to the correct path COPY --from=builder \ diff --git a/Scripts/build-release b/Scripts/build-release new file mode 100755 index 00000000..baaf6c76 --- /dev/null +++ b/Scripts/build-release @@ -0,0 +1,19 @@ +#!/bin/sh + +set -e +cd "$(dirname $0)/.." + +extra_flags=() + +if [ -n "$TARGETARCH" ]; then + case "$TARGETARCH" in + amd64) extra_flags=(--arch x86_64);; + arm64) extra_flags=(--arch arm64);; + *) + echo "Unsupported target arch '$TARGETARCH'" + exit 1 + ;; + esac +fi + +exec swift build -c release "${extra_flags[@]}" diff --git a/Scripts/setup-dotbuild-tree b/Scripts/setup-dotbuild-tree new file mode 100755 index 00000000..db8f4fd0 --- /dev/null +++ b/Scripts/setup-dotbuild-tree @@ -0,0 +1,40 @@ +#!/bin/sh + +set -e +cd "$(dirname $0)/.." + +if [ -n "$TARGETOS" ]; then + case "$TARGETOS" in + linux) + vendor=unknown + os=linux-gnu + ;; + *) + echo "Unsupported target os '$TARGETOS'" + exit 1 + ;; + esac +else + echo "Please specify the target OS by setting the TARGETOS variable!" + exit 1 +fi + +if [ -n "$TARGETARCH" ]; then + case "$TARGETARCH" in + amd64) arch=x86_64;; + arm64) arch=arm64;; + *) + echo "Unsupported target arch '${D2TARGETARCH[@]}'" + exit 1 + ;; + esac +else + echo "Please specify the target arch by setting the TARGETARCH variable!" + exit 1 +fi + +config=release + +cd .build +mkdir -p "$arch-$vendor-$os"/"$config" +ln -s "$config" "$arch-$vendor-$os"/"$config" From 96c18d193e4155dde362f681f3f2354201bfd76e Mon Sep 17 00:00:00 2001 From: fwcd Date: Thu, 29 Dec 2022 19:47:43 +0100 Subject: [PATCH 04/65] Build arm64 image in CI again --- .github/workflows/docker.yml | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 05f27bbe..47611c66 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -26,36 +26,6 @@ jobs: uses: docker/build-push-action@v2 with: context: . - # TODO: Ideally we could just build an aarch64 image too (under QEMU user-mode emulation) by using - # - # platforms: linux/amd64,linux/arm64/v8 - # - # While this works for most build steps, this unfortunately fails with obscure - # 'invalid manifest' errors when building D2 with the Swift Package Manager under QEMU. - # Further investigation revealed that these crashes occur due to `SIGTRAP`s sent to `swift-driver` while - # fetching/checking out D2's dependencies. This can be reproduced by building the image - # up to the step before `swift build -c release` and then manually running the Swift build e.g. in a - # shell within such a container. The errors will occur and QEMU will place a core dump - # (`*.core`) in the working directory (which can be inspected using `lldb /usr/bin/swift-driver --core ` - # and the command `bt all`). - # - # Possible ways to resolve this issue: - # - # - Use a GitHub-provisioned native ARM runner once available - # - Try using full system emulation for aarch64 with QEMU with Docker - # - It might not be trivial to set this up, most documentation seems to be focused - # on running QEMU under user-mode emulation when building multiarch images - # - Docker Desktop seemed to do something in this direction on M1 Macs prior to using the Virtualization framework - # - A drackback would be that this is probably quite a bit slower - # - https://reverseengineering.stackexchange.com/questions/15015/qemu-gdb-server-thread-problem - # seems to deal with a similar problem. - # - Investigate whether we can cross-compile Swift from x86_64 directly within the Docker build - # - This would probably be the best (and fastest) approach, but might be non-trivial, - # depending on how well the Swift Package Manager supports cross-compiling from x86_64 Linux hosts to - # aarch64 Linux. - # - https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/ - # is a guide that explains how to do this (in general) - # - https://github.com/rvolosatovs/docker-protobuf/pull/91 does something similar - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64/v8 push: true tags: ghcr.io/fwcd/d2:latest From c4eac68325f45a365a37beaf230da64a5e59e173 Mon Sep 17 00:00:00 2001 From: fwcd Date: Thu, 29 Dec 2022 19:54:39 +0100 Subject: [PATCH 05/65] Force scripts to run in bash --- Scripts/build-release | 2 +- Scripts/setup-dotbuild-tree | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Scripts/build-release b/Scripts/build-release index baaf6c76..7c6a68ea 100755 --- a/Scripts/build-release +++ b/Scripts/build-release @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -e cd "$(dirname $0)/.." diff --git a/Scripts/setup-dotbuild-tree b/Scripts/setup-dotbuild-tree index db8f4fd0..91478fd8 100755 --- a/Scripts/setup-dotbuild-tree +++ b/Scripts/setup-dotbuild-tree @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -e cd "$(dirname $0)/.." From 7d73cead2090fe2eb4c5cd40aa70129591039826 Mon Sep 17 00:00:00 2001 From: fwcd Date: Thu, 29 Dec 2022 20:00:32 +0100 Subject: [PATCH 06/65] Pass TARGETOS and TARGETARCH from Dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c79e2ce7..d775b350 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ COPY Sources Sources COPY Tests Tests COPY Package.swift Package.resolved ./ COPY Scripts/build-release Scripts/ -RUN Scripts/build-release +RUN TARGETARCH=$TARGETARCH Scripts/build-release FROM swift:5.7-focal-slim AS runner @@ -36,7 +36,7 @@ COPY LICENSE README.md ./ # Set up .build folder in runner WORKDIR /opt/d2/.build COPY Scripts/setup-dotbuild-tree Scripts/ -RUN Scripts/setup-dotbuild-tree +RUN TARGETOS=$TARGETOS TARGETARCH=$TARGETARCH Scripts/setup-dotbuild-tree # Copy font used by swiftplot to the correct path COPY --from=builder \ From 3b6f5084bb79758f4bb982567175a9224318e8e6 Mon Sep 17 00:00:00 2001 From: fwcd Date: Thu, 29 Dec 2022 20:09:00 +0100 Subject: [PATCH 07/65] Declare TARGETOS and TARGETARCH as ARGs --- Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index d775b350..0076db56 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,8 @@ FROM --platform=$BUILDPLATFORM swift:5.7-focal AS builder COPY Scripts/install-build-dependencies-apt Scripts/ RUN Scripts/install-build-dependencies-apt && rm -rf /var/lib/apt/lists/* +ARG TARGETARCH + # Build WORKDIR /opt/d2 COPY Sources Sources @@ -33,6 +35,9 @@ RUN Scripts/install-node-dependencies COPY Resources Resources COPY LICENSE README.md ./ +ARG TARGETOS +ARG TARGETARCH + # Set up .build folder in runner WORKDIR /opt/d2/.build COPY Scripts/setup-dotbuild-tree Scripts/ From 105873adb1620297f2962e98a71fc6514a33045b Mon Sep 17 00:00:00 2001 From: fwcd Date: Thu, 29 Dec 2022 20:14:41 +0100 Subject: [PATCH 08/65] Make sure .build is always created --- Scripts/setup-dotbuild-tree | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Scripts/setup-dotbuild-tree b/Scripts/setup-dotbuild-tree index 91478fd8..0338b9eb 100755 --- a/Scripts/setup-dotbuild-tree +++ b/Scripts/setup-dotbuild-tree @@ -35,6 +35,7 @@ fi config=release +mkdir -p .build/"$arch-$vendor-$os"/"$config" + cd .build -mkdir -p "$arch-$vendor-$os"/"$config" ln -s "$config" "$arch-$vendor-$os"/"$config" From 9329e1235f0a06191d4a840ac6591be36978d2e9 Mon Sep 17 00:00:00 2001 From: fwcd Date: Thu, 29 Dec 2022 21:49:28 +0100 Subject: [PATCH 09/65] Set up include and linker paths for crosscompiling --- Dockerfile | 14 ++++++++++---- Scripts/build-release | 18 ++++++++++++++++-- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0076db56..f4241007 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,24 @@ FROM --platform=$BUILDPLATFORM swift:5.7-focal AS builder +ARG BUILDARCH +ARG TARGETARCH + +# Install cross compilation toolchain if needed +RUN if [ "$BUILDARCH" != "$TARGETARCH" ]; then \ + apt-get update && apt-get install -y crossbuild-essential-$TARGETARCH \ +fi + # Install native dependencies COPY Scripts/install-build-dependencies-apt Scripts/ RUN Scripts/install-build-dependencies-apt && rm -rf /var/lib/apt/lists/* -ARG TARGETARCH - # Build WORKDIR /opt/d2 COPY Sources Sources COPY Tests Tests COPY Package.swift Package.resolved ./ COPY Scripts/build-release Scripts/ -RUN TARGETARCH=$TARGETARCH Scripts/build-release +RUN DEBIAN=ON Scripts/build-release FROM swift:5.7-focal-slim AS runner @@ -41,7 +47,7 @@ ARG TARGETARCH # Set up .build folder in runner WORKDIR /opt/d2/.build COPY Scripts/setup-dotbuild-tree Scripts/ -RUN TARGETOS=$TARGETOS TARGETARCH=$TARGETARCH Scripts/setup-dotbuild-tree +RUN Scripts/setup-dotbuild-tree # Copy font used by swiftplot to the correct path COPY --from=builder \ diff --git a/Scripts/build-release b/Scripts/build-release index 7c6a68ea..4090aa2b 100755 --- a/Scripts/build-release +++ b/Scripts/build-release @@ -7,8 +7,14 @@ extra_flags=() if [ -n "$TARGETARCH" ]; then case "$TARGETARCH" in - amd64) extra_flags=(--arch x86_64);; - arm64) extra_flags=(--arch arm64);; + amd64) + extra_flags+=(--arch x86_64) + debian_arch=x86_64 + ;; + arm64) + extra_flags+=(--arch arm64) + debian_arch=aarch64 + ;; *) echo "Unsupported target arch '$TARGETARCH'" exit 1 @@ -16,4 +22,12 @@ if [ -n "$TARGETARCH" ]; then esac fi +if [ "$BUILDARCH" != "$TARGETARCH" ] && [ -n "$DEBIAN" ]; then + triplet="$debian_arch-linux-gnu" + extra_flags+=( + -Xcc -I"/usr/include/$triplet" + -Xlinker -L"/usr/lib/$triplet" + ) +fi + exec swift build -c release "${extra_flags[@]}" From 7460baf65c6608ec1229f372f1686389278d886a Mon Sep 17 00:00:00 2001 From: fwcd Date: Thu, 29 Dec 2022 21:53:29 +0100 Subject: [PATCH 10/65] Use single-line if for installing crossbuild sysroot --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index f4241007..951e7efa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,9 +4,7 @@ ARG BUILDARCH ARG TARGETARCH # Install cross compilation toolchain if needed -RUN if [ "$BUILDARCH" != "$TARGETARCH" ]; then \ - apt-get update && apt-get install -y crossbuild-essential-$TARGETARCH \ -fi +RUN if [ "$BUILDARCH" != "$TARGETARCH" ]; then apt-get update && apt-get install -y crossbuild-essential-$TARGETARCH; fi # Install native dependencies COPY Scripts/install-build-dependencies-apt Scripts/ From 1a688369f9e4aa51a6c319ed318c38ed8c69a906 Mon Sep 17 00:00:00 2001 From: fwcd Date: Thu, 29 Dec 2022 22:24:34 +0100 Subject: [PATCH 11/65] Move cross-compilation setup into dependencies script --- Dockerfile | 3 --- Scripts/build-release | 8 +++--- Scripts/install-build-dependencies-apt | 34 ++++++++++++++++++++------ 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 951e7efa..f8633475 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,6 @@ FROM --platform=$BUILDPLATFORM swift:5.7-focal AS builder ARG BUILDARCH ARG TARGETARCH -# Install cross compilation toolchain if needed -RUN if [ "$BUILDARCH" != "$TARGETARCH" ]; then apt-get update && apt-get install -y crossbuild-essential-$TARGETARCH; fi - # Install native dependencies COPY Scripts/install-build-dependencies-apt Scripts/ RUN Scripts/install-build-dependencies-apt && rm -rf /var/lib/apt/lists/* diff --git a/Scripts/build-release b/Scripts/build-release index 4090aa2b..2928f558 100755 --- a/Scripts/build-release +++ b/Scripts/build-release @@ -9,11 +9,11 @@ if [ -n "$TARGETARCH" ]; then case "$TARGETARCH" in amd64) extra_flags+=(--arch x86_64) - debian_arch=x86_64 + arch_name=x86_64 ;; arm64) extra_flags+=(--arch arm64) - debian_arch=aarch64 + arch_name=aarch64 ;; *) echo "Unsupported target arch '$TARGETARCH'" @@ -22,8 +22,8 @@ if [ -n "$TARGETARCH" ]; then esac fi -if [ "$BUILDARCH" != "$TARGETARCH" ] && [ -n "$DEBIAN" ]; then - triplet="$debian_arch-linux-gnu" +if [ -n "$BUILDARCH" ] && [ -n "$TARGETARCH" ] && [ "$BUILDARCH" != "$TARGETARCH" ] && [ -n "$DEBIAN" ]; then + triplet="$arch_name-linux-gnu" extra_flags+=( -Xcc -I"/usr/include/$triplet" -Xlinker -L"/usr/lib/$triplet" diff --git a/Scripts/install-build-dependencies-apt b/Scripts/install-build-dependencies-apt index 4fe63b27..75366127 100755 --- a/Scripts/install-build-dependencies-apt +++ b/Scripts/install-build-dependencies-apt @@ -5,12 +5,32 @@ set -e cd "$(dirname $0)/.." -apt-get update -apt-get install -y \ - libcairo2-dev \ - libsqlite3-dev \ - libleptonica-dev \ - libtesseract-dev \ +dependencies=( + libcairo2-dev + libsqlite3-dev + libleptonica-dev + libtesseract-dev libgraphviz-dev + # TODO: poppler-utils and libssl1.1 needed? +) + +# Set up cross-compilation if needed +if [ -n "$BUILDARCH" ] && [ -n "$TARGETARCH" ] && [ "$BUILDARCH" != "$TARGETARCH" ]; then + # Install sysroot for the target architecture + apt-get update + apt-get install -y \ + crossbuild-essential-"$TARGETARCH" + + # Configure dpkg to install packages for target architecture too + dpkg --add-architecture "$TARGETARCH" -# TODO: poppler-utils and libssl1.1 needed? + # Suffix dependencies with architecture + for (( i=0; i<${#dependencies[@]}; i++ )); do + dependencies[$i]="${dependencies[$i]}:$TARGETARCH" + done +fi + +# Install the dependencies +apt-get update +apt-get install -y \ + "${dependencies[@]}" From 006e1f0433c037e3d74bba94adcdad00a75ae71f Mon Sep 17 00:00:00 2001 From: fwcd Date: Fri, 30 Dec 2022 00:27:24 +0100 Subject: [PATCH 12/65] Install the native target toolchain to get the Swift libraries ...for cross-compilation. --- Dockerfile | 13 +++++- Scripts/build-release | 35 +++++++++++--- Scripts/install-build-dependencies-apt | 9 ++-- Scripts/install-cross-target-toolchain-focal | 49 ++++++++++++++++++++ 4 files changed, 94 insertions(+), 12 deletions(-) create mode 100755 Scripts/install-cross-target-toolchain-focal diff --git a/Dockerfile b/Dockerfile index f8633475..a474406c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,7 @@ -FROM --platform=$BUILDPLATFORM swift:5.7-focal AS builder +ARG SWIFTVERSION 5.7.2 +ARG UBUNTUDISTRO focal + +FROM --platform=$BUILDPLATFORM swift:$SWIFTVERSION-$UBUNTUDISTRO AS builder ARG BUILDARCH ARG TARGETARCH @@ -7,6 +10,12 @@ ARG TARGETARCH COPY Scripts/install-build-dependencies-apt Scripts/ RUN Scripts/install-build-dependencies-apt && rm -rf /var/lib/apt/lists/* +# Install the target toolchain for cross-compilation if needed +# (only required for locating e.g. Swift standard library modules, +# the actual compilation is done with the installed host Swift toolchain) +COPY Scripts/install-cross-target-toolchain-$UBUNTUDISTRO Scripts/ +RUN Scripts/install-cross-target-toolchain-$UBUNTUDISTRO + # Build WORKDIR /opt/d2 COPY Sources Sources @@ -15,7 +24,7 @@ COPY Package.swift Package.resolved ./ COPY Scripts/build-release Scripts/ RUN DEBIAN=ON Scripts/build-release -FROM swift:5.7-focal-slim AS runner +FROM swift:$SWIFTVERSION-$UBUNTUDISTRO-slim AS runner # Install Curl, add-apt-repository and node package repository RUN apt-get update && apt-get install -y curl software-properties-common && rm -rf /var/lib/apt/lists/* diff --git a/Scripts/build-release b/Scripts/build-release index 2928f558..7ef35f81 100755 --- a/Scripts/build-release +++ b/Scripts/build-release @@ -3,9 +3,13 @@ set -e cd "$(dirname $0)/.." +# Set up cross-compilation if needed + +cross_target_toolchains_dir="$(pwd)/local/cross-target-toolchains" extra_flags=() if [ -n "$TARGETARCH" ]; then + echo "==> Setting flags for target arch '$TARGETARCH'" case "$TARGETARCH" in amd64) extra_flags+=(--arch x86_64) @@ -22,12 +26,31 @@ if [ -n "$TARGETARCH" ]; then esac fi -if [ -n "$BUILDARCH" ] && [ -n "$TARGETARCH" ] && [ "$BUILDARCH" != "$TARGETARCH" ] && [ -n "$DEBIAN" ]; then - triplet="$arch_name-linux-gnu" - extra_flags+=( - -Xcc -I"/usr/include/$triplet" - -Xlinker -L"/usr/lib/$triplet" - ) +if [ -n "$BUILDARCH" ] && [ -n "$TARGETARCH" ] && [ "$BUILDARCH" != "$TARGETARCH" ]; then + if [ -n "$DEBIAN" ]; then + echo "==> Adding cross-compilation sysroot to include and linker path" + triplet="$arch_name-linux-gnu" + extra_flags+=( + -Xcc -I"/usr/include/$triplet" + -Xlinker -L"/usr/lib/$triplet" + ) + fi + + if [ -n "$SWIFTVERSION" ]; then + cross_target_toolchain_dir="$cross_target_toolchains_dir/$TARGETARCH-$SWIFTVERSION" + + echo "==> Checking $cross_target_toolchain_dir for cross target toolchain" + if [ -d "$cross_target_toolchain_dir" ]; then + echo "==> Adding cross target toolchain $cross_target_toolchain_dir to include path" + extra_flags+=( + -Xswiftc -I"$cross_target_toolchain_dir/usr/lib/swift" + ) + else + echo "Warning: Not adding cross target toolchain since $cross_target_toolchain_dir does not exist! You probably have to run Scripts/install-cross-target-toolchain-* first (if applicable)." + fi + else + echo "Warning: Not adding cross target toolchain since SWIFTVERSION is not set!" + fi fi exec swift build -c release "${extra_flags[@]}" diff --git a/Scripts/install-build-dependencies-apt b/Scripts/install-build-dependencies-apt index 75366127..e7b8dba4 100755 --- a/Scripts/install-build-dependencies-apt +++ b/Scripts/install-build-dependencies-apt @@ -15,22 +15,23 @@ dependencies=( ) # Set up cross-compilation if needed + if [ -n "$BUILDARCH" ] && [ -n "$TARGETARCH" ] && [ "$BUILDARCH" != "$TARGETARCH" ]; then - # Install sysroot for the target architecture + echo "==> Installing cross-compilation sysroot for $TARGETARCH" apt-get update apt-get install -y \ crossbuild-essential-"$TARGETARCH" - # Configure dpkg to install packages for target architecture too + echo "==> Configuring dpkg to install packages for $TARGETARCH" dpkg --add-architecture "$TARGETARCH" - # Suffix dependencies with architecture + echo "==> Setting dependencies to use $TARGETARCH" for (( i=0; i<${#dependencies[@]}; i++ )); do dependencies[$i]="${dependencies[$i]}:$TARGETARCH" done fi -# Install the dependencies +echo "==> Installing ${dependencies[@]}" apt-get update apt-get install -y \ "${dependencies[@]}" diff --git a/Scripts/install-cross-target-toolchain-focal b/Scripts/install-cross-target-toolchain-focal new file mode 100755 index 00000000..9a10824a --- /dev/null +++ b/Scripts/install-cross-target-toolchain-focal @@ -0,0 +1,49 @@ +#!/bin/bash + +# Downloads a target toolchain when cross-compiling. We use the (already installed) host compiler +# but use the libraries from the downloaded toolchain. + +set -e +cd "$(dirname $0)/.." + +downloads_dir="$(pwd)/local/downloads" +toolchains_dir="$(pwd)/local/cross-target-toolchains" +flags_path="$toolchains_dir/build-flags.txt" + +for var in BUILDARCH TARGETARCH SWIFTVERSION; do + if [ -z ${!var} ]; then + echo "Please set the variable $var!" + exit 1 + fi +done + +case "$TARGETARCH" in + amd64) arch_name=x86_64;; + arm64) arch_name=aarch64;; + *) + echo "Unsupported target arch '$TARGETARCH'" + exit 1 + ;; +esac + +if [ "$BUILDARCH" != "$TARGETARCH" ]; then + archive_name="swift-$SWIFTVERSION-RELEASE-ubuntu20.04-$arch_name.tar.gz" + archive_url="https://download.swift.org/swift-$SWIFTVERSION-release/ubuntu2004-$arch_name/swift-$SWIFTVERSION-RELEASE/$archive_name" + toolchain_dir="$toolchains_dir/$TARGETARCH-$SWIFTVERSION" + + echo "==> Setting up local directories" + mkdir -p "$downloads_dir" + mkdir -p "$toolchain_dir" + + cd "$downloads_dir" + + if [ ! -f "$archive_name" ]; then + echo "==> Downloading Swift toolchain for $TARGETARCH" + curl -OL "$archive_url" + fi + + if [ ! -d "$toolchain_dir" ]; then + echo "==> Unpacking Swift toolchain for $TARGETARCH into $toolchain_dir" + tar -xvf "$archive_name" --strip-components 1 -C "$toolchain_dir" + fi +fi From 56f4e6b893a82a1f1c64c4929626e8cb239317b6 Mon Sep 17 00:00:00 2001 From: fwcd Date: Fri, 30 Dec 2022 00:36:27 +0100 Subject: [PATCH 13/65] Fix ARG syntax --- Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index a474406c..5454c1ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ -ARG SWIFTVERSION 5.7.2 -ARG UBUNTUDISTRO focal +ARG SWIFTVERSION=5.7.2 +ARG UBUNTUDISTRO=focal -FROM --platform=$BUILDPLATFORM swift:$SWIFTVERSION-$UBUNTUDISTRO AS builder +FROM --platform=$BUILDPLATFORM swift:${SWIFTVERSION}-${UBUNTUDISTRO} AS builder ARG BUILDARCH ARG TARGETARCH @@ -13,8 +13,8 @@ RUN Scripts/install-build-dependencies-apt && rm -rf /var/lib/apt/lists/* # Install the target toolchain for cross-compilation if needed # (only required for locating e.g. Swift standard library modules, # the actual compilation is done with the installed host Swift toolchain) -COPY Scripts/install-cross-target-toolchain-$UBUNTUDISTRO Scripts/ -RUN Scripts/install-cross-target-toolchain-$UBUNTUDISTRO +COPY Scripts/install-cross-target-toolchain-${UBUNTUDISTRO} Scripts/ +RUN Scripts/install-cross-target-toolchain-${UBUNTUDISTRO} # Build WORKDIR /opt/d2 @@ -24,7 +24,7 @@ COPY Package.swift Package.resolved ./ COPY Scripts/build-release Scripts/ RUN DEBIAN=ON Scripts/build-release -FROM swift:$SWIFTVERSION-$UBUNTUDISTRO-slim AS runner +FROM swift:${SWIFTVERSION}-${UBUNTUDISTRO}-slim AS runner # Install Curl, add-apt-repository and node package repository RUN apt-get update && apt-get install -y curl software-properties-common && rm -rf /var/lib/apt/lists/* From ed4807c59bce0862a7f0190a58e861609b2bfa77 Mon Sep 17 00:00:00 2001 From: fwcd Date: Fri, 30 Dec 2022 00:38:34 +0100 Subject: [PATCH 14/65] Renew ARG for second stage --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 5454c1ec..9d0d0a6b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,8 @@ ARG UBUNTUDISTRO=focal FROM --platform=$BUILDPLATFORM swift:${SWIFTVERSION}-${UBUNTUDISTRO} AS builder +ARG SWIFTVERSION +ARG UBUNTUDISTRO ARG BUILDARCH ARG TARGETARCH From 388ad62858e381d3c05c84f7631e3c4ddde1480b Mon Sep 17 00:00:00 2001 From: fwcd Date: Fri, 30 Dec 2022 01:11:57 +0100 Subject: [PATCH 15/65] Add unified script for setting up cross sysroot with qemu-debootstrap --- Dockerfile | 9 ++--- Scripts/build-release | 26 +++---------- Scripts/install-build-dependencies-apt | 20 +--------- Scripts/install-cross-compilation-sysroot | 45 +++++++++++++++++++++++ Scripts/install-runtime-dependencies-apt | 16 ++++---- 5 files changed, 64 insertions(+), 52 deletions(-) create mode 100755 Scripts/install-cross-compilation-sysroot diff --git a/Dockerfile b/Dockerfile index 9d0d0a6b..d6de76a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,11 +12,10 @@ ARG TARGETARCH COPY Scripts/install-build-dependencies-apt Scripts/ RUN Scripts/install-build-dependencies-apt && rm -rf /var/lib/apt/lists/* -# Install the target toolchain for cross-compilation if needed -# (only required for locating e.g. Swift standard library modules, -# the actual compilation is done with the installed host Swift toolchain) -COPY Scripts/install-cross-target-toolchain-${UBUNTUDISTRO} Scripts/ -RUN Scripts/install-cross-target-toolchain-${UBUNTUDISTRO} +# Set up cross compilation sysroot if needed +ARG CROSSCOMPILESYSROOT=/usr/${TARGETARCH}-ubuntu-${UBUNTUDISTRO} +COPY Scripts/install-cross-compilation-sysroot Scripts/ +RUN if [ "$BUILDARCH" != "$TARGETARCH" ]; then Scripts/install-cross-compilation-sysroot fi # Build WORKDIR /opt/d2 diff --git a/Scripts/build-release b/Scripts/build-release index 7ef35f81..ec1460ca 100755 --- a/Scripts/build-release +++ b/Scripts/build-release @@ -24,32 +24,16 @@ if [ -n "$TARGETARCH" ]; then exit 1 ;; esac -fi -if [ -n "$BUILDARCH" ] && [ -n "$TARGETARCH" ] && [ "$BUILDARCH" != "$TARGETARCH" ]; then - if [ -n "$DEBIAN" ]; then - echo "==> Adding cross-compilation sysroot to include and linker path" + if [ -n "$BUILDARCH" ] && [ "$BUILDARCH" != "$TARGETARCH" ] && [ -n "$CROSSCOMPILESYSROOT" ]; then + echo "==> Adding cross-compilation sysroot $CROSSCOMPILESYSROOT to C compilation paths" triplet="$arch_name-linux-gnu" extra_flags+=( - -Xcc -I"/usr/include/$triplet" - -Xlinker -L"/usr/lib/$triplet" + -Xcc --sysroot="$CROSSCOMPILESYSROOT/usr/include" + # TODO: Linker? ) - fi - - if [ -n "$SWIFTVERSION" ]; then - cross_target_toolchain_dir="$cross_target_toolchains_dir/$TARGETARCH-$SWIFTVERSION" - echo "==> Checking $cross_target_toolchain_dir for cross target toolchain" - if [ -d "$cross_target_toolchain_dir" ]; then - echo "==> Adding cross target toolchain $cross_target_toolchain_dir to include path" - extra_flags+=( - -Xswiftc -I"$cross_target_toolchain_dir/usr/lib/swift" - ) - else - echo "Warning: Not adding cross target toolchain since $cross_target_toolchain_dir does not exist! You probably have to run Scripts/install-cross-target-toolchain-* first (if applicable)." - fi - else - echo "Warning: Not adding cross target toolchain since SWIFTVERSION is not set!" + # TODO: Swift toolchain fi fi diff --git a/Scripts/install-build-dependencies-apt b/Scripts/install-build-dependencies-apt index e7b8dba4..82a38ea1 100755 --- a/Scripts/install-build-dependencies-apt +++ b/Scripts/install-build-dependencies-apt @@ -14,24 +14,6 @@ dependencies=( # TODO: poppler-utils and libssl1.1 needed? ) -# Set up cross-compilation if needed - -if [ -n "$BUILDARCH" ] && [ -n "$TARGETARCH" ] && [ "$BUILDARCH" != "$TARGETARCH" ]; then - echo "==> Installing cross-compilation sysroot for $TARGETARCH" - apt-get update - apt-get install -y \ - crossbuild-essential-"$TARGETARCH" - - echo "==> Configuring dpkg to install packages for $TARGETARCH" - dpkg --add-architecture "$TARGETARCH" - - echo "==> Setting dependencies to use $TARGETARCH" - for (( i=0; i<${#dependencies[@]}; i++ )); do - dependencies[$i]="${dependencies[$i]}:$TARGETARCH" - done -fi - echo "==> Installing ${dependencies[@]}" apt-get update -apt-get install -y \ - "${dependencies[@]}" +apt-get install -y "${dependencies[@]}" diff --git a/Scripts/install-cross-compilation-sysroot b/Scripts/install-cross-compilation-sysroot new file mode 100755 index 00000000..0fe3a11c --- /dev/null +++ b/Scripts/install-cross-compilation-sysroot @@ -0,0 +1,45 @@ +#!/bin/bash + +# Sets up a sysroot for cross-compilation on Ubuntu for Ubuntu. + +set -e +cd "$(dirname $0)/.." + +if [ -z "$CROSSCOMPILESYSROOT" ]; then + echo "Please point CROSSCOMPILESYSROOT to the directory to which the sysroot will be installed!" + exit 1 +fi + +if [ -z "$TARGETARCH" ]; then + echo "Please set TARGETARCH to a target architecture to bootstrap the sysroot for!" + exit 1 +fi + +if [ -z "$UBUNTUDISTRO" ]; then + echo "Please set UBUNTUDISTRO to the short name of a Ubuntu version, e.g. 'focal'." + exit 1 +fi + +if ! command -v apt-get &>/dev/null; then + echo "Please make to be running a Debian-based distro and to have apt-get on your PATH!" + exit 1 +fi + +echo "==> Installing qemu-debootstrap..." +apt-get update +apt-get install -y debootstrap qemu-user-static schroot + +echo "==> Bootstrapping sysroot..." +qemu-debootstrap --arch="$TARGETARCH" "$UBUNTUDISTRO" "$CROSSCOMPILESYSROOT" + +echo "==> Copying D2 scripts into sysroot..." +sysroot_d2_dir="/opt/d2" +mkdir -p "$CROSSCOMPILESYSROOT/$sysroot_d2_dir" +cp -r ./Scripts "$CROSSCOMPILESYSROOT/$sysroot_d2_dir/" + +echo "==> Chrooting into sysroot and installing D2's dependencies..." +schroot -c "$CROSSCOMPILESYSROOT" +cd "$sysroot_d2_dir" +Scripts/install-build-dependencies-apt + +# TODO: Install Swift too (and replace the current cross target toolchain mechanism) diff --git a/Scripts/install-runtime-dependencies-apt b/Scripts/install-runtime-dependencies-apt index 0f6a7897..0ed91fe4 100755 --- a/Scripts/install-runtime-dependencies-apt +++ b/Scripts/install-runtime-dependencies-apt @@ -5,12 +5,14 @@ set -e cd "$(dirname $0)/.." -apt-get update -apt-get install -y \ - libcairo2 \ - libsqlite3-0 \ - tesseract-ocr \ - graphviz \ +dependencies=( + libcairo2 + libsqlite3-0 + tesseract-ocr + graphviz nodejs + # TODO: poppler-utils and libssl1.1 needed? +) -# TODO: poppler-utils and libssl1.1 needed? +apt-get update +apt-get install -y "${dependencies[@]}" From 83b946c0451461a5add2fc96330e7f012bf6ce25 Mon Sep 17 00:00:00 2001 From: fwcd Date: Fri, 30 Dec 2022 01:14:57 +0100 Subject: [PATCH 16/65] Fix typo --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d6de76a8..9ef4e484 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN Scripts/install-build-dependencies-apt && rm -rf /var/lib/apt/lists/* # Set up cross compilation sysroot if needed ARG CROSSCOMPILESYSROOT=/usr/${TARGETARCH}-ubuntu-${UBUNTUDISTRO} COPY Scripts/install-cross-compilation-sysroot Scripts/ -RUN if [ "$BUILDARCH" != "$TARGETARCH" ]; then Scripts/install-cross-compilation-sysroot fi +RUN if [ "$BUILDARCH" != "$TARGETARCH" ]; then Scripts/install-cross-compilation-sysroot; fi # Build WORKDIR /opt/d2 From 08a770a3877940acade43d520e4fadee0be97f41 Mon Sep 17 00:00:00 2001 From: fwcd Date: Fri, 30 Dec 2022 01:33:38 +0100 Subject: [PATCH 17/65] Set up schroot configuration --- Scripts/install-cross-compilation-sysroot | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Scripts/install-cross-compilation-sysroot b/Scripts/install-cross-compilation-sysroot index 0fe3a11c..bb1cae0a 100755 --- a/Scripts/install-cross-compilation-sysroot +++ b/Scripts/install-cross-compilation-sysroot @@ -37,8 +37,16 @@ sysroot_d2_dir="/opt/d2" mkdir -p "$CROSSCOMPILESYSROOT/$sysroot_d2_dir" cp -r ./Scripts "$CROSSCOMPILESYSROOT/$sysroot_d2_dir/" -echo "==> Chrooting into sysroot and installing D2's dependencies..." -schroot -c "$CROSSCOMPILESYSROOT" +echo "==> Setting up schroot..." +echo "[cross-compile-sysroot] +description=Cross-compile sysroot +directory=$CROSSCOMPILESYSROOT +root-users=$(whoami) +users=$(whoami) +type=directory" > /etc/schroot/chroot.d/cross-compile-sysroot + +echo "==> Schrooting into sysroot and installing D2's dependencies..." +schroot -c cross-compile-sysroot cd "$sysroot_d2_dir" Scripts/install-build-dependencies-apt From 85f1ff9170cf9414569134f32afa8bb2e8d1f576 Mon Sep 17 00:00:00 2001 From: fwcd Date: Fri, 30 Dec 2022 02:10:25 +0100 Subject: [PATCH 18/65] Chroot directly into sysroot --- Scripts/install-cross-compilation-sysroot | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/Scripts/install-cross-compilation-sysroot b/Scripts/install-cross-compilation-sysroot index bb1cae0a..8cc76c6e 100755 --- a/Scripts/install-cross-compilation-sysroot +++ b/Scripts/install-cross-compilation-sysroot @@ -37,16 +37,8 @@ sysroot_d2_dir="/opt/d2" mkdir -p "$CROSSCOMPILESYSROOT/$sysroot_d2_dir" cp -r ./Scripts "$CROSSCOMPILESYSROOT/$sysroot_d2_dir/" -echo "==> Setting up schroot..." -echo "[cross-compile-sysroot] -description=Cross-compile sysroot -directory=$CROSSCOMPILESYSROOT -root-users=$(whoami) -users=$(whoami) -type=directory" > /etc/schroot/chroot.d/cross-compile-sysroot - -echo "==> Schrooting into sysroot and installing D2's dependencies..." -schroot -c cross-compile-sysroot +echo "==> Chrooting into sysroot and installing D2's dependencies..." +chroot "$CROSSCOMPILESYSROOT" cd "$sysroot_d2_dir" Scripts/install-build-dependencies-apt From 4b6703a99ae420215a53d19ca7b48cbf44d381dc Mon Sep 17 00:00:00 2001 From: fwcd Date: Fri, 30 Dec 2022 02:33:57 +0100 Subject: [PATCH 19/65] Output the new sysroot for debugging --- Scripts/install-cross-compilation-sysroot | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Scripts/install-cross-compilation-sysroot b/Scripts/install-cross-compilation-sysroot index 8cc76c6e..3282327d 100755 --- a/Scripts/install-cross-compilation-sysroot +++ b/Scripts/install-cross-compilation-sysroot @@ -35,7 +35,15 @@ qemu-debootstrap --arch="$TARGETARCH" "$UBUNTUDISTRO" "$CROSSCOMPILESYSROOT" echo "==> Copying D2 scripts into sysroot..." sysroot_d2_dir="/opt/d2" mkdir -p "$CROSSCOMPILESYSROOT/$sysroot_d2_dir" -cp -r ./Scripts "$CROSSCOMPILESYSROOT/$sysroot_d2_dir/" +cp -R ./Scripts "$CROSSCOMPILESYSROOT/$sysroot_d2_dir/" + +output-for-debugging() { + echo "$1: $(cd "$1" && echo *)" +} + +output-for-debugging "$CROSSCOMPILESYSROOT" +output-for-debugging "$CROSSCOMPILESYSROOT/opt" +output-for-debugging "$CROSSCOMPILESYSROOT/opt/d2" echo "==> Chrooting into sysroot and installing D2's dependencies..." chroot "$CROSSCOMPILESYSROOT" From 40e3664727fb5dadbeae22f12636192ce41db2ab Mon Sep 17 00:00:00 2001 From: fwcd Date: Fri, 30 Dec 2022 02:53:33 +0100 Subject: [PATCH 20/65] Script chroot properly --- Scripts/install-cross-compilation-sysroot | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Scripts/install-cross-compilation-sysroot b/Scripts/install-cross-compilation-sysroot index 3282327d..d2b9ad83 100755 --- a/Scripts/install-cross-compilation-sysroot +++ b/Scripts/install-cross-compilation-sysroot @@ -46,8 +46,9 @@ output-for-debugging "$CROSSCOMPILESYSROOT/opt" output-for-debugging "$CROSSCOMPILESYSROOT/opt/d2" echo "==> Chrooting into sysroot and installing D2's dependencies..." -chroot "$CROSSCOMPILESYSROOT" +cat << EOF | chroot "$CROSSCOMPILESYSROOT" cd "$sysroot_d2_dir" Scripts/install-build-dependencies-apt +EOF # TODO: Install Swift too (and replace the current cross target toolchain mechanism) From 8483e7463349c29ee41746562f5520a2ae161800 Mon Sep 17 00:00:00 2001 From: fwcd Date: Fri, 30 Dec 2022 03:09:59 +0100 Subject: [PATCH 21/65] Remove unused DEBIAN=ON flag --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9ef4e484..319d998f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ COPY Sources Sources COPY Tests Tests COPY Package.swift Package.resolved ./ COPY Scripts/build-release Scripts/ -RUN DEBIAN=ON Scripts/build-release +RUN Scripts/build-release FROM swift:${SWIFTVERSION}-${UBUNTUDISTRO}-slim AS runner From c7d881ee944749471d84cbb40a49581c36776b35 Mon Sep 17 00:00:00 2001 From: fwcd Date: Fri, 30 Dec 2022 03:15:34 +0100 Subject: [PATCH 22/65] Enable repositories in install-build-dependencies-apt --- Scripts/install-build-dependencies-apt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Scripts/install-build-dependencies-apt b/Scripts/install-build-dependencies-apt index 82a38ea1..5a2294da 100755 --- a/Scripts/install-build-dependencies-apt +++ b/Scripts/install-build-dependencies-apt @@ -14,6 +14,11 @@ dependencies=( # TODO: poppler-utils and libssl1.1 needed? ) +echo "==> Enabling all repositories" +for repo in main universe multiverse; do + add-apt-repository $repo +done + echo "==> Installing ${dependencies[@]}" apt-get update apt-get install -y "${dependencies[@]}" From 2089be765d169426310fe9363b7acf211c07e7cb Mon Sep 17 00:00:00 2001 From: fwcd Date: Fri, 30 Dec 2022 03:18:46 +0100 Subject: [PATCH 23/65] Install add-apt-repository before using it --- Scripts/install-build-dependencies-apt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Scripts/install-build-dependencies-apt b/Scripts/install-build-dependencies-apt index 5a2294da..47434154 100755 --- a/Scripts/install-build-dependencies-apt +++ b/Scripts/install-build-dependencies-apt @@ -14,6 +14,10 @@ dependencies=( # TODO: poppler-utils and libssl1.1 needed? ) +echo "==> Installing add-apt-repository" +apt-get update +apt-get install -y software-properties-common + echo "==> Enabling all repositories" for repo in main universe multiverse; do add-apt-repository $repo From e82b04c1324b0d1f6a01c0d29c262c0186dcff71 Mon Sep 17 00:00:00 2001 From: fwcd Date: Fri, 30 Dec 2022 03:45:24 +0100 Subject: [PATCH 24/65] Fix sysroot path --- Scripts/build-release | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/build-release b/Scripts/build-release index ec1460ca..1120ab4d 100755 --- a/Scripts/build-release +++ b/Scripts/build-release @@ -29,7 +29,7 @@ if [ -n "$TARGETARCH" ]; then echo "==> Adding cross-compilation sysroot $CROSSCOMPILESYSROOT to C compilation paths" triplet="$arch_name-linux-gnu" extra_flags+=( - -Xcc --sysroot="$CROSSCOMPILESYSROOT/usr/include" + -Xcc --sysroot="$CROSSCOMPILESYSROOT" # TODO: Linker? ) From 9a11317b692e3f469f66817d0bd45df34aef4ff2 Mon Sep 17 00:00:00 2001 From: fwcd Date: Fri, 30 Dec 2022 04:10:14 +0100 Subject: [PATCH 25/65] Set cross compile sysroot paths manually again --- Scripts/build-release | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Scripts/build-release b/Scripts/build-release index 1120ab4d..de9ea3cd 100755 --- a/Scripts/build-release +++ b/Scripts/build-release @@ -29,8 +29,8 @@ if [ -n "$TARGETARCH" ]; then echo "==> Adding cross-compilation sysroot $CROSSCOMPILESYSROOT to C compilation paths" triplet="$arch_name-linux-gnu" extra_flags+=( - -Xcc --sysroot="$CROSSCOMPILESYSROOT" - # TODO: Linker? + -Xcc -I"$CROSSCOMPILESYSROOT/usr/include" + -Xlinker -L"$CROSSCOMPILESYSROOT/usr/lib" ) # TODO: Swift toolchain From 2caeeeb3c9c504e92bcebac272b110ac4c227f26 Mon Sep 17 00:00:00 2001 From: fwcd Date: Fri, 30 Dec 2022 04:37:16 +0100 Subject: [PATCH 26/65] Add build-essential to APT dependencies --- Scripts/install-build-dependencies-apt | 1 + 1 file changed, 1 insertion(+) diff --git a/Scripts/install-build-dependencies-apt b/Scripts/install-build-dependencies-apt index 47434154..e41cd759 100755 --- a/Scripts/install-build-dependencies-apt +++ b/Scripts/install-build-dependencies-apt @@ -6,6 +6,7 @@ set -e cd "$(dirname $0)/.." dependencies=( + build-essential libcairo2-dev libsqlite3-dev libleptonica-dev From 532c4770c23fb5e96c42f8ce7a9dd931aeb7506c Mon Sep 17 00:00:00 2001 From: fwcd Date: Fri, 30 Dec 2022 04:37:53 +0100 Subject: [PATCH 27/65] Output extra flags --- Scripts/build-release | 1 + 1 file changed, 1 insertion(+) diff --git a/Scripts/build-release b/Scripts/build-release index de9ea3cd..1835767e 100755 --- a/Scripts/build-release +++ b/Scripts/build-release @@ -37,4 +37,5 @@ if [ -n "$TARGETARCH" ]; then fi fi +echo "==> Building D2 with flags ${extra_flags[@]}" exec swift build -c release "${extra_flags[@]}" From ca5de6594af270ec76127d2e2b875c3b1fa7ec3b Mon Sep 17 00:00:00 2001 From: fwcd Date: Fri, 30 Dec 2022 05:07:21 +0100 Subject: [PATCH 28/65] Set up registry caching for CI-built images See https://docs.docker.com/build/cache/backends/registry/ --- .github/workflows/docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 47611c66..9ac891f3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -29,3 +29,5 @@ jobs: platforms: linux/amd64,linux/arm64/v8 push: true tags: ghcr.io/fwcd/d2:latest + cache-from: type=registry,ref=ghcr.io/fwcd/d2-buildcache:buildcache + cache-to: type=registry,ref=ghcr.io/fwcd/d2-buildcache:buildcache,mode=max From 478e4fe1ad0aa36a9127e5ac316a66d4b85bdd9f Mon Sep 17 00:00:00 2001 From: fwcd Date: Fri, 30 Dec 2022 16:58:03 +0100 Subject: [PATCH 29/65] Consolidate install and sysroot into single step This also avoid unnecessarily installing the dependencies for both the build and the target architecture when only the target architecture is needed. --- Dockerfile | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 319d998f..b6c62d3e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,14 +8,11 @@ ARG UBUNTUDISTRO ARG BUILDARCH ARG TARGETARCH -# Install native dependencies -COPY Scripts/install-build-dependencies-apt Scripts/ -RUN Scripts/install-build-dependencies-apt && rm -rf /var/lib/apt/lists/* - -# Set up cross compilation sysroot if needed ARG CROSSCOMPILESYSROOT=/usr/${TARGETARCH}-ubuntu-${UBUNTUDISTRO} -COPY Scripts/install-cross-compilation-sysroot Scripts/ -RUN if [ "$BUILDARCH" != "$TARGETARCH" ]; then Scripts/install-cross-compilation-sysroot; fi + +# Install native dependencies or (if cross-compiling) set up sysroot +COPY Scripts/install-build-dependencies-apt Scripts/install-cross-compilation-sysroot Scripts/ +RUN if [ "$BUILDARCH" == "$TARGETARCH" ]; then Scripts/install-build-dependencies-apt && rm -rf /var/lib/apt/lists/* else Scripts/install-cross-compilation-sysroot; fi # Build WORKDIR /opt/d2 From 38ac19aea227ce069a8f266b372a92d08dd9f2bf Mon Sep 17 00:00:00 2001 From: fwcd Date: Fri, 30 Dec 2022 18:02:59 +0100 Subject: [PATCH 30/65] Fix typo --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b6c62d3e..a2732eda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ ARG CROSSCOMPILESYSROOT=/usr/${TARGETARCH}-ubuntu-${UBUNTUDISTRO} # Install native dependencies or (if cross-compiling) set up sysroot COPY Scripts/install-build-dependencies-apt Scripts/install-cross-compilation-sysroot Scripts/ -RUN if [ "$BUILDARCH" == "$TARGETARCH" ]; then Scripts/install-build-dependencies-apt && rm -rf /var/lib/apt/lists/* else Scripts/install-cross-compilation-sysroot; fi +RUN if [ "$BUILDARCH" == "$TARGETARCH" ]; then Scripts/install-build-dependencies-apt && rm -rf /var/lib/apt/lists/*; else Scripts/install-cross-compilation-sysroot; fi # Build WORKDIR /opt/d2 From e69a7745e7ab4ebac5e450acea9cb3bfc92bea51 Mon Sep 17 00:00:00 2001 From: fwcd Date: Fri, 30 Dec 2022 18:10:29 +0100 Subject: [PATCH 31/65] Fix double equals operator in Dockerfile Some shells seem to only recognize the single-equals variant --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a2732eda..af6a9692 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ ARG CROSSCOMPILESYSROOT=/usr/${TARGETARCH}-ubuntu-${UBUNTUDISTRO} # Install native dependencies or (if cross-compiling) set up sysroot COPY Scripts/install-build-dependencies-apt Scripts/install-cross-compilation-sysroot Scripts/ -RUN if [ "$BUILDARCH" == "$TARGETARCH" ]; then Scripts/install-build-dependencies-apt && rm -rf /var/lib/apt/lists/*; else Scripts/install-cross-compilation-sysroot; fi +RUN if [ "$BUILDARCH" = "$TARGETARCH" ]; then Scripts/install-build-dependencies-apt && rm -rf /var/lib/apt/lists/*; else Scripts/install-cross-compilation-sysroot; fi # Build WORKDIR /opt/d2 From 3c6f2ae1d72291ee6a63a92b423ede29627ff860 Mon Sep 17 00:00:00 2001 From: fwcd Date: Mon, 2 Jan 2023 00:51:20 +0100 Subject: [PATCH 32/65] Pass cross-compile include path to C++ compiler too --- Scripts/build-release | 1 + 1 file changed, 1 insertion(+) diff --git a/Scripts/build-release b/Scripts/build-release index 1835767e..3eb3085f 100755 --- a/Scripts/build-release +++ b/Scripts/build-release @@ -30,6 +30,7 @@ if [ -n "$TARGETARCH" ]; then triplet="$arch_name-linux-gnu" extra_flags+=( -Xcc -I"$CROSSCOMPILESYSROOT/usr/include" + -Xcxx -I"$CROSSCOMPILESYSROOT/usr/include" -Xlinker -L"$CROSSCOMPILESYSROOT/usr/lib" ) From 662c62fc4ca4f27a01431bdec3262616a721c369 Mon Sep 17 00:00:00 2001 From: fwcd Date: Mon, 2 Jan 2023 01:35:23 +0100 Subject: [PATCH 33/65] Add OS check to install-cross-compilation-sysroot --- Scripts/install-cross-compilation-sysroot | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Scripts/install-cross-compilation-sysroot b/Scripts/install-cross-compilation-sysroot index d2b9ad83..a8956255 100755 --- a/Scripts/install-cross-compilation-sysroot +++ b/Scripts/install-cross-compilation-sysroot @@ -5,6 +5,11 @@ set -e cd "$(dirname $0)/.." +if [ "$(uname -s)" != "Linux" ]; then + echo "Creating cross-compilation sysroots is only supported on Linux!" + exit 1 +fi + if [ -z "$CROSSCOMPILESYSROOT" ]; then echo "Please point CROSSCOMPILESYSROOT to the directory to which the sysroot will be installed!" exit 1 From b3f459b2f4b746edd5826b65b3b27f1ec25f76dc Mon Sep 17 00:00:00 2001 From: fwcd Date: Mon, 2 Jan 2023 01:37:36 +0100 Subject: [PATCH 34/65] Use debootstrap directly instead of qemu-debootstrap qemu-debootstrap seems to be deprecated --- Scripts/install-cross-compilation-sysroot | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Scripts/install-cross-compilation-sysroot b/Scripts/install-cross-compilation-sysroot index a8956255..223e0cea 100755 --- a/Scripts/install-cross-compilation-sysroot +++ b/Scripts/install-cross-compilation-sysroot @@ -30,12 +30,19 @@ if ! command -v apt-get &>/dev/null; then exit 1 fi -echo "==> Installing qemu-debootstrap..." +# See https://wiki.ubuntu.com/ARM/RootfsFromScratch/QemuDebootstrap + +echo "==> Installing debootstrap..." apt-get update -apt-get install -y debootstrap qemu-user-static schroot +apt-get install -y debootstrap qemu-user-static echo "==> Bootstrapping sysroot..." -qemu-debootstrap --arch="$TARGETARCH" "$UBUNTUDISTRO" "$CROSSCOMPILESYSROOT" +debootstrap \ + --verbose \ + --foreign \ + --arch="$TARGETARCH" \ + "$UBUNTUDISTRO" \ + "$CROSSCOMPILESYSROOT" echo "==> Copying D2 scripts into sysroot..." sysroot_d2_dir="/opt/d2" From f9ebc4e85f5ffde9f9bbe915b2aa522209cc9f64 Mon Sep 17 00:00:00 2001 From: fwcd Date: Mon, 2 Jan 2023 01:53:43 +0100 Subject: [PATCH 35/65] Add script for mapping arch names --- Scripts/build-release | 16 ++-------------- Scripts/get-linux-arch-name | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 14 deletions(-) create mode 100755 Scripts/get-linux-arch-name diff --git a/Scripts/build-release b/Scripts/build-release index 3eb3085f..19e82f39 100755 --- a/Scripts/build-release +++ b/Scripts/build-release @@ -10,20 +10,8 @@ extra_flags=() if [ -n "$TARGETARCH" ]; then echo "==> Setting flags for target arch '$TARGETARCH'" - case "$TARGETARCH" in - amd64) - extra_flags+=(--arch x86_64) - arch_name=x86_64 - ;; - arm64) - extra_flags+=(--arch arm64) - arch_name=aarch64 - ;; - *) - echo "Unsupported target arch '$TARGETARCH'" - exit 1 - ;; - esac + arch_name="$(Scripts/get-linux-arch-name $TARGETARCH)" + extra_flags+=(--arch "$arch_name") if [ -n "$BUILDARCH" ] && [ "$BUILDARCH" != "$TARGETARCH" ] && [ -n "$CROSSCOMPILESYSROOT" ]; then echo "==> Adding cross-compilation sysroot $CROSSCOMPILESYSROOT to C compilation paths" diff --git a/Scripts/get-linux-arch-name b/Scripts/get-linux-arch-name new file mode 100755 index 00000000..973297e0 --- /dev/null +++ b/Scripts/get-linux-arch-name @@ -0,0 +1,18 @@ +#!/bin/bash + +# Unfortunately, Docker uses a different name for some CPU architectures than some +# Linux tools (arm64 vs aarch64, amd64 vs x86_64, ...), therefore we use this script +# to perform this mapping once and for all. + +set -e + +if [ $# -lt 1 ]; then + echo "Usage: $0 [arch name]" + exit 1 +fi + +case "$1" in + amd64) echo "x86_64";; + arm64) echo "aarch64";; + *) echo "$1";; +esac From deb61d1e313034993cc927795312429d39d07db5 Mon Sep 17 00:00:00 2001 From: fwcd Date: Mon, 2 Jan 2023 01:56:23 +0100 Subject: [PATCH 36/65] Perform second-stage bootstrap too --- Scripts/install-cross-compilation-sysroot | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Scripts/install-cross-compilation-sysroot b/Scripts/install-cross-compilation-sysroot index 223e0cea..67104ce8 100755 --- a/Scripts/install-cross-compilation-sysroot +++ b/Scripts/install-cross-compilation-sysroot @@ -36,7 +36,7 @@ echo "==> Installing debootstrap..." apt-get update apt-get install -y debootstrap qemu-user-static -echo "==> Bootstrapping sysroot..." +echo "==> Performing first-stage bootstrap..." debootstrap \ --verbose \ --foreign \ @@ -44,6 +44,12 @@ debootstrap \ "$UBUNTUDISTRO" \ "$CROSSCOMPILESYSROOT" +echo "==> Performing second-stage bootstrap with QEMU..." +cp "/usr/bin/qemu-$(Scripts/get-linux-arch-name $TARGETARCH)-static" "$CROSSCOMPILESYSROOT/usr/bin" +cat << EOF | chroot "$CROSSCOMPILESYSROOT" +/debootstrap/debootstrap --second-stage +EOF + echo "==> Copying D2 scripts into sysroot..." sysroot_d2_dir="/opt/d2" mkdir -p "$CROSSCOMPILESYSROOT/$sysroot_d2_dir" From 01ca499f0581cf9459e3ce0c24a2a478f0931a38 Mon Sep 17 00:00:00 2001 From: fwcd Date: Mon, 2 Jan 2023 03:32:53 +0100 Subject: [PATCH 37/65] Copy get-linux-arch-name to image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index af6a9692..92db681b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ WORKDIR /opt/d2 COPY Sources Sources COPY Tests Tests COPY Package.swift Package.resolved ./ -COPY Scripts/build-release Scripts/ +COPY Scripts/get-linux-arch-name Scripts/build-release Scripts/ RUN Scripts/build-release FROM swift:${SWIFTVERSION}-${UBUNTUDISTRO}-slim AS runner From aa3f4e10eaa2baefa2eecbb2cabed06a468a7edf Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 21 Feb 2023 12:31:53 +0100 Subject: [PATCH 38/65] Add pull request trigger to Docker workflow (without push) --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9ac891f3..f4378fdb 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,7 +4,7 @@ on: push: branches: - main - - docker/** + pull_request: workflow_dispatch: jobs: From e588b87f8348e65f5aa1f385e0064cceff1d6401 Mon Sep 17 00:00:00 2001 From: fwcd Date: Sun, 30 Jul 2023 17:44:59 +0200 Subject: [PATCH 39/65] Bump to 5.8.1-jammy --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e79e7581..a2aff386 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -ARG SWIFTVERSION=5.7 -ARG UBUNTUDISTRO=focal +ARG SWIFTVERSION=5.8.1 +ARG UBUNTUDISTRO=jammy FROM --platform=$BUILDPLATFORM swift:${SWIFTVERSION}-${UBUNTUDISTRO} AS builder From d021efb795480351d247be9ed8f7d3266f66643d Mon Sep 17 00:00:00 2001 From: fwcd Date: Sun, 30 Jul 2023 17:57:57 +0200 Subject: [PATCH 40/65] Copy get-linux-arch-name in earlier step already --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a2aff386..cc8023c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ ARG TARGETARCH ARG CROSSCOMPILESYSROOT=/usr/${TARGETARCH}-ubuntu-${UBUNTUDISTRO} # Install native dependencies or (if cross-compiling) set up sysroot -COPY Scripts/install-build-dependencies-apt Scripts/install-cross-compilation-sysroot Scripts/ +COPY Scripts/install-build-dependencies-apt Scripts/install-cross-compilation-sysroot Scripts/get-linux-arch-name Scripts/ RUN if [ "$BUILDARCH" = "$TARGETARCH" ]; then Scripts/install-build-dependencies-apt && rm -rf /var/lib/apt/lists/*; else Scripts/install-cross-compilation-sysroot; fi # Build @@ -19,7 +19,7 @@ WORKDIR /opt/d2 COPY Sources Sources COPY Tests Tests COPY Package.swift Package.resolved ./ -COPY Scripts/get-linux-arch-name Scripts/build-release Scripts/ +COPY Scripts/build-release Scripts/ RUN Scripts/build-release FROM swift:${SWIFTVERSION}-${UBUNTUDISTRO}-slim AS runner From 92bf5119cd5e2056ebd39c3a2d53710e97fd9746 Mon Sep 17 00:00:00 2001 From: fwcd Date: Sun, 30 Jul 2023 18:02:39 +0200 Subject: [PATCH 41/65] Set workdir earlier --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cc8023c9..c0712156 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,12 +10,13 @@ ARG TARGETARCH ARG CROSSCOMPILESYSROOT=/usr/${TARGETARCH}-ubuntu-${UBUNTUDISTRO} +WORKDIR /opt/d2 + # Install native dependencies or (if cross-compiling) set up sysroot COPY Scripts/install-build-dependencies-apt Scripts/install-cross-compilation-sysroot Scripts/get-linux-arch-name Scripts/ RUN if [ "$BUILDARCH" = "$TARGETARCH" ]; then Scripts/install-build-dependencies-apt && rm -rf /var/lib/apt/lists/*; else Scripts/install-cross-compilation-sysroot; fi # Build -WORKDIR /opt/d2 COPY Sources Sources COPY Tests Tests COPY Package.swift Package.resolved ./ From be190dd600f21ba81bab7f62f044a8c439de8695 Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 01:04:40 +0200 Subject: [PATCH 42/65] Use multi-stage build to create sysroot --- Dockerfile | 20 ++++++++++++++------ Scripts/build-release | 27 +++++++++++++++------------ 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index c0712156..8e1a428f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,14 @@ ARG SWIFTVERSION=5.8.1 ARG UBUNTUDISTRO=jammy +FROM swift:${SWIFTVERSION}-${UBUNTUDISTRO} AS sysroot + +WORKDIR /opt/d2 + +# Install build dependencies into target sysroot +COPY Scripts/install-build-dependencies-apt Scripts/ +RUN Scripts/install-build-dependencies-apt && rm -rf /var/lib/apt/lists/* + FROM --platform=$BUILDPLATFORM swift:${SWIFTVERSION}-${UBUNTUDISTRO} AS builder ARG SWIFTVERSION @@ -8,19 +16,19 @@ ARG UBUNTUDISTRO ARG BUILDARCH ARG TARGETARCH -ARG CROSSCOMPILESYSROOT=/usr/${TARGETARCH}-ubuntu-${UBUNTUDISTRO} +ARG TARGETSYSROOT=/usr/${TARGETARCH}-ubuntu-${UBUNTUDISTRO} WORKDIR /opt/d2 -# Install native dependencies or (if cross-compiling) set up sysroot -COPY Scripts/install-build-dependencies-apt Scripts/install-cross-compilation-sysroot Scripts/get-linux-arch-name Scripts/ -RUN if [ "$BUILDARCH" = "$TARGETARCH" ]; then Scripts/install-build-dependencies-apt && rm -rf /var/lib/apt/lists/*; else Scripts/install-cross-compilation-sysroot; fi +# Copy target sysroot into builder +# TODO: Only copy stuff that we need for compilation (/usr/lib, /usr/include etc.) +COPY --from=sysroot / ${TARGETSYSROOT} -# Build +# (Cross-)compile D2 COPY Sources Sources COPY Tests Tests COPY Package.swift Package.resolved ./ -COPY Scripts/build-release Scripts/ +COPY Scripts/build-release Scripts/get-linux-arch-name Scripts/ RUN Scripts/build-release FROM swift:${SWIFTVERSION}-${UBUNTUDISTRO}-slim AS runner diff --git a/Scripts/build-release b/Scripts/build-release index 19e82f39..d0887697 100755 --- a/Scripts/build-release +++ b/Scripts/build-release @@ -5,25 +5,28 @@ cd "$(dirname $0)/.." # Set up cross-compilation if needed -cross_target_toolchains_dir="$(pwd)/local/cross-target-toolchains" extra_flags=() if [ -n "$TARGETARCH" ]; then echo "==> Setting flags for target arch '$TARGETARCH'" arch_name="$(Scripts/get-linux-arch-name $TARGETARCH)" - extra_flags+=(--arch "$arch_name") - if [ -n "$BUILDARCH" ] && [ "$BUILDARCH" != "$TARGETARCH" ] && [ -n "$CROSSCOMPILESYSROOT" ]; then - echo "==> Adding cross-compilation sysroot $CROSSCOMPILESYSROOT to C compilation paths" - triplet="$arch_name-linux-gnu" - extra_flags+=( - -Xcc -I"$CROSSCOMPILESYSROOT/usr/include" - -Xcxx -I"$CROSSCOMPILESYSROOT/usr/include" - -Xlinker -L"$CROSSCOMPILESYSROOT/usr/lib" - ) - - # TODO: Swift toolchain + # Workaround for https://github.com/stephencelis/CSQLite/pull/1 + if [ ! -f /usr/include/sqlite3.h ]; then + ln -s {"$TARGETSYSROOT",}/usr/include/sqlite3.h fi + + extra_flags+=( + --arch "$arch_name" + --sdk "$TARGETSYSROOT" + # TODO: Figure out which of these flags are redundant + -Xswiftc -resource-dir -Xswiftc "$TARGETSYSROOT/usr/lib/swift" + -Xcc -I"$TARGETSYSROOT/usr/include" + -Xcc -I"$TARGETSYSROOT/usr/include/freetype2" + -Xcc -I"$TARGETSYSROOT/usr/include/cairo" + -Xcxx -I"$TARGETSYSROOT/usr/include" + -Xlinker -L"$TARGETSYSROOT/usr/lib" + ) fi echo "==> Building D2 with flags ${extra_flags[@]}" From 974cd92013304616f9a370abb443320f6dae12b7 Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 01:05:59 +0200 Subject: [PATCH 43/65] Remove legacy cross-compile sysroot scripts --- Scripts/install-cross-compilation-sysroot | 72 -------------------- Scripts/install-cross-target-toolchain-focal | 49 ------------- 2 files changed, 121 deletions(-) delete mode 100755 Scripts/install-cross-compilation-sysroot delete mode 100755 Scripts/install-cross-target-toolchain-focal diff --git a/Scripts/install-cross-compilation-sysroot b/Scripts/install-cross-compilation-sysroot deleted file mode 100755 index 67104ce8..00000000 --- a/Scripts/install-cross-compilation-sysroot +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash - -# Sets up a sysroot for cross-compilation on Ubuntu for Ubuntu. - -set -e -cd "$(dirname $0)/.." - -if [ "$(uname -s)" != "Linux" ]; then - echo "Creating cross-compilation sysroots is only supported on Linux!" - exit 1 -fi - -if [ -z "$CROSSCOMPILESYSROOT" ]; then - echo "Please point CROSSCOMPILESYSROOT to the directory to which the sysroot will be installed!" - exit 1 -fi - -if [ -z "$TARGETARCH" ]; then - echo "Please set TARGETARCH to a target architecture to bootstrap the sysroot for!" - exit 1 -fi - -if [ -z "$UBUNTUDISTRO" ]; then - echo "Please set UBUNTUDISTRO to the short name of a Ubuntu version, e.g. 'focal'." - exit 1 -fi - -if ! command -v apt-get &>/dev/null; then - echo "Please make to be running a Debian-based distro and to have apt-get on your PATH!" - exit 1 -fi - -# See https://wiki.ubuntu.com/ARM/RootfsFromScratch/QemuDebootstrap - -echo "==> Installing debootstrap..." -apt-get update -apt-get install -y debootstrap qemu-user-static - -echo "==> Performing first-stage bootstrap..." -debootstrap \ - --verbose \ - --foreign \ - --arch="$TARGETARCH" \ - "$UBUNTUDISTRO" \ - "$CROSSCOMPILESYSROOT" - -echo "==> Performing second-stage bootstrap with QEMU..." -cp "/usr/bin/qemu-$(Scripts/get-linux-arch-name $TARGETARCH)-static" "$CROSSCOMPILESYSROOT/usr/bin" -cat << EOF | chroot "$CROSSCOMPILESYSROOT" -/debootstrap/debootstrap --second-stage -EOF - -echo "==> Copying D2 scripts into sysroot..." -sysroot_d2_dir="/opt/d2" -mkdir -p "$CROSSCOMPILESYSROOT/$sysroot_d2_dir" -cp -R ./Scripts "$CROSSCOMPILESYSROOT/$sysroot_d2_dir/" - -output-for-debugging() { - echo "$1: $(cd "$1" && echo *)" -} - -output-for-debugging "$CROSSCOMPILESYSROOT" -output-for-debugging "$CROSSCOMPILESYSROOT/opt" -output-for-debugging "$CROSSCOMPILESYSROOT/opt/d2" - -echo "==> Chrooting into sysroot and installing D2's dependencies..." -cat << EOF | chroot "$CROSSCOMPILESYSROOT" -cd "$sysroot_d2_dir" -Scripts/install-build-dependencies-apt -EOF - -# TODO: Install Swift too (and replace the current cross target toolchain mechanism) diff --git a/Scripts/install-cross-target-toolchain-focal b/Scripts/install-cross-target-toolchain-focal deleted file mode 100755 index 9a10824a..00000000 --- a/Scripts/install-cross-target-toolchain-focal +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -# Downloads a target toolchain when cross-compiling. We use the (already installed) host compiler -# but use the libraries from the downloaded toolchain. - -set -e -cd "$(dirname $0)/.." - -downloads_dir="$(pwd)/local/downloads" -toolchains_dir="$(pwd)/local/cross-target-toolchains" -flags_path="$toolchains_dir/build-flags.txt" - -for var in BUILDARCH TARGETARCH SWIFTVERSION; do - if [ -z ${!var} ]; then - echo "Please set the variable $var!" - exit 1 - fi -done - -case "$TARGETARCH" in - amd64) arch_name=x86_64;; - arm64) arch_name=aarch64;; - *) - echo "Unsupported target arch '$TARGETARCH'" - exit 1 - ;; -esac - -if [ "$BUILDARCH" != "$TARGETARCH" ]; then - archive_name="swift-$SWIFTVERSION-RELEASE-ubuntu20.04-$arch_name.tar.gz" - archive_url="https://download.swift.org/swift-$SWIFTVERSION-release/ubuntu2004-$arch_name/swift-$SWIFTVERSION-RELEASE/$archive_name" - toolchain_dir="$toolchains_dir/$TARGETARCH-$SWIFTVERSION" - - echo "==> Setting up local directories" - mkdir -p "$downloads_dir" - mkdir -p "$toolchain_dir" - - cd "$downloads_dir" - - if [ ! -f "$archive_name" ]; then - echo "==> Downloading Swift toolchain for $TARGETARCH" - curl -OL "$archive_url" - fi - - if [ ! -d "$toolchain_dir" ]; then - echo "==> Unpacking Swift toolchain for $TARGETARCH into $toolchain_dir" - tar -xvf "$archive_name" --strip-components 1 -C "$toolchain_dir" - fi -fi From 8c07d2037e00da27baf53e5c13a9dc4d57c9c21d Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 01:14:43 +0200 Subject: [PATCH 44/65] Add $arch_name-linux-gnu to linker path --- Scripts/build-release | 1 + 1 file changed, 1 insertion(+) diff --git a/Scripts/build-release b/Scripts/build-release index d0887697..8831a6cf 100755 --- a/Scripts/build-release +++ b/Scripts/build-release @@ -26,6 +26,7 @@ if [ -n "$TARGETARCH" ]; then -Xcc -I"$TARGETSYSROOT/usr/include/cairo" -Xcxx -I"$TARGETSYSROOT/usr/include" -Xlinker -L"$TARGETSYSROOT/usr/lib" + -Xlinker -L"$TARGETSYSROOT/usr/lib/$arch_name-linux-gnu" # libtesseract installs there ) fi From 7e0dcf2be55a847b375bc526e5f6b9d8d517f26d Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 03:13:34 +0200 Subject: [PATCH 45/65] Work around glibc header issue --- Scripts/build-release | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Scripts/build-release b/Scripts/build-release index 8831a6cf..4e2cbdc1 100755 --- a/Scripts/build-release +++ b/Scripts/build-release @@ -16,15 +16,24 @@ if [ -n "$TARGETARCH" ]; then ln -s {"$TARGETSYSROOT",}/usr/include/sqlite3.h fi + # There seem to be some strange errors with (glibc) + # when including the native /usr/include on an x86_64 host, + # therefore we'll only include it when cross-compiling. + if [ "$BUILDARCH" != "$TARGETARCH" ]; then + extra_flags+=( + -Xcc -I"$TARGETSYSROOT/usr/include" + -Xcc -I"$TARGETSYSROOT/usr/include/$arch_name-linux-gnu" + -Xcxx -I"$TARGETSYSROOT/usr/include" + ) + fi + extra_flags+=( --arch "$arch_name" --sdk "$TARGETSYSROOT" # TODO: Figure out which of these flags are redundant -Xswiftc -resource-dir -Xswiftc "$TARGETSYSROOT/usr/lib/swift" - -Xcc -I"$TARGETSYSROOT/usr/include" -Xcc -I"$TARGETSYSROOT/usr/include/freetype2" -Xcc -I"$TARGETSYSROOT/usr/include/cairo" - -Xcxx -I"$TARGETSYSROOT/usr/include" -Xlinker -L"$TARGETSYSROOT/usr/lib" -Xlinker -L"$TARGETSYSROOT/usr/lib/$arch_name-linux-gnu" # libtesseract installs there ) From abdafad49fdeec7a5bc135bd87b575f004503b46 Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 04:11:54 +0200 Subject: [PATCH 46/65] Work around issue --- Scripts/build-release | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/Scripts/build-release b/Scripts/build-release index 4e2cbdc1..99aa8cec 100755 --- a/Scripts/build-release +++ b/Scripts/build-release @@ -17,14 +17,10 @@ if [ -n "$TARGETARCH" ]; then fi # There seem to be some strange errors with (glibc) - # when including the native /usr/include on an x86_64 host, - # therefore we'll only include it when cross-compiling. - if [ "$BUILDARCH" != "$TARGETARCH" ]; then - extra_flags+=( - -Xcc -I"$TARGETSYSROOT/usr/include" - -Xcc -I"$TARGETSYSROOT/usr/include/$arch_name-linux-gnu" - -Xcxx -I"$TARGETSYSROOT/usr/include" - ) + # on host + target x86_64. Since this header seems to be provided + # elsewhere, we can remove it from the target sysroot as a workaround. + if [ -n "$TARGETSYSROOT" ] && [ "$BUILDARCH" = amd64 ] && [ "$TARGETARCH" = amd64 ]; then + rm "$TARGETSYSROOT/usr/include/tgmath.h" fi extra_flags+=( @@ -32,6 +28,7 @@ if [ -n "$TARGETARCH" ]; then --sdk "$TARGETSYSROOT" # TODO: Figure out which of these flags are redundant -Xswiftc -resource-dir -Xswiftc "$TARGETSYSROOT/usr/lib/swift" + -Xcc -I"$TARGETSYSROOT/usr/include" -Xcc -I"$TARGETSYSROOT/usr/include/freetype2" -Xcc -I"$TARGETSYSROOT/usr/include/cairo" -Xlinker -L"$TARGETSYSROOT/usr/lib" From 684ba142f0774f0949859ed8d2d4b035ec875e0d Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 04:13:38 +0200 Subject: [PATCH 47/65] Install libstdc++-12-dev as build dependency --- Scripts/install-build-dependencies-apt | 1 + 1 file changed, 1 insertion(+) diff --git a/Scripts/install-build-dependencies-apt b/Scripts/install-build-dependencies-apt index e41cd759..dfa9700a 100755 --- a/Scripts/install-build-dependencies-apt +++ b/Scripts/install-build-dependencies-apt @@ -12,6 +12,7 @@ dependencies=( libleptonica-dev libtesseract-dev libgraphviz-dev + libstdc++-12-dev # TODO: poppler-utils and libssl1.1 needed? ) From 6a15b904a5fea71254870c8cd6e9ff5af5de04f5 Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 04:17:04 +0200 Subject: [PATCH 48/65] Use lld as linker --- Scripts/build-release | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Scripts/build-release b/Scripts/build-release index 99aa8cec..335b2ec9 100755 --- a/Scripts/build-release +++ b/Scripts/build-release @@ -26,8 +26,9 @@ if [ -n "$TARGETARCH" ]; then extra_flags+=( --arch "$arch_name" --sdk "$TARGETSYSROOT" - # TODO: Figure out which of these flags are redundant + -Xswiftc -use-ld=lld -Xswiftc -resource-dir -Xswiftc "$TARGETSYSROOT/usr/lib/swift" + # TODO: Figure out which of these flags are redundant -Xcc -I"$TARGETSYSROOT/usr/include" -Xcc -I"$TARGETSYSROOT/usr/include/freetype2" -Xcc -I"$TARGETSYSROOT/usr/include/cairo" From 51e6b577b4876046cb610fca8f3c32516a8fb7dc Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 04:30:02 +0200 Subject: [PATCH 49/65] Include standard headers from target sysroot --- Scripts/build-release | 1 + 1 file changed, 1 insertion(+) diff --git a/Scripts/build-release b/Scripts/build-release index 335b2ec9..8044134a 100755 --- a/Scripts/build-release +++ b/Scripts/build-release @@ -31,6 +31,7 @@ if [ -n "$TARGETARCH" ]; then # TODO: Figure out which of these flags are redundant -Xcc -I"$TARGETSYSROOT/usr/include" -Xcc -I"$TARGETSYSROOT/usr/include/freetype2" + -Xcc -I"$TARGETSYSROOT/usr/include/$arch_name-linux-gnu" -Xcc -I"$TARGETSYSROOT/usr/include/cairo" -Xlinker -L"$TARGETSYSROOT/usr/lib" -Xlinker -L"$TARGETSYSROOT/usr/lib/$arch_name-linux-gnu" # libtesseract installs there From 3cd4021a9823c29ba706f3d9d3fc39590e26c396 Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 15:23:40 +0200 Subject: [PATCH 50/65] Place target sysroot under /usr/local ...in order to not interfere with cross-GCC packages. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8e1a428f..3a29052e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ ARG UBUNTUDISTRO ARG BUILDARCH ARG TARGETARCH -ARG TARGETSYSROOT=/usr/${TARGETARCH}-ubuntu-${UBUNTUDISTRO} +ARG TARGETSYSROOT=/usr/local/${TARGETARCH}-ubuntu-${UBUNTUDISTRO} WORKDIR /opt/d2 From 80e2a07a4c1eb1e1895cfe787e266f07e7843382 Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 15:24:23 +0200 Subject: [PATCH 51/65] Check for $TARGETSYSROOT variable --- Scripts/build-release | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Scripts/build-release b/Scripts/build-release index 8044134a..cac1c142 100755 --- a/Scripts/build-release +++ b/Scripts/build-release @@ -7,7 +7,7 @@ cd "$(dirname $0)/.." extra_flags=() -if [ -n "$TARGETARCH" ]; then +if [ -n "$TARGETARCH" ] && [ -n "$TARGETSYSROOT" ]; then echo "==> Setting flags for target arch '$TARGETARCH'" arch_name="$(Scripts/get-linux-arch-name $TARGETARCH)" @@ -19,8 +19,8 @@ if [ -n "$TARGETARCH" ]; then # There seem to be some strange errors with (glibc) # on host + target x86_64. Since this header seems to be provided # elsewhere, we can remove it from the target sysroot as a workaround. - if [ -n "$TARGETSYSROOT" ] && [ "$BUILDARCH" = amd64 ] && [ "$TARGETARCH" = amd64 ]; then - rm "$TARGETSYSROOT/usr/include/tgmath.h" + if [ "$BUILDARCH" = amd64 ] && [ "$TARGETARCH" = amd64 ] && [ -f "$TARGETSYSROOT/usr/include/tgmath.h" ]; then + mv "$TARGETSYSROOT/usr/include/tgmath.h{,.backup}" fi extra_flags+=( From 8863b3a9acf1f02eb21860467c7d92a6ee375035 Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 15:50:26 +0200 Subject: [PATCH 52/65] Install cross-GCC instead of installing it natively to the sysroot This seems to work better with the Swift cross-linker (lld), since we now have a -tools-directory we can point it at. --- Dockerfile | 6 +++- Scripts/build-release | 18 ++-------- Scripts/install-build-dependencies-apt | 1 - Scripts/prepare-docker-buildroot | 49 ++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 17 deletions(-) create mode 100755 Scripts/prepare-docker-buildroot diff --git a/Dockerfile b/Dockerfile index 3a29052e..d102b3ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,11 +24,15 @@ WORKDIR /opt/d2 # TODO: Only copy stuff that we need for compilation (/usr/lib, /usr/include etc.) COPY --from=sysroot / ${TARGETSYSROOT} +# Install (cross-)GCC and patch some paths +COPY Scripts/prepare-docker-buildroot Scripts/get-linux-arch-name Scripts/ +RUN Scripts/prepare-docker-buildroot + # (Cross-)compile D2 COPY Sources Sources COPY Tests Tests COPY Package.swift Package.resolved ./ -COPY Scripts/build-release Scripts/get-linux-arch-name Scripts/ +COPY Scripts/build-release Scripts/ RUN Scripts/build-release FROM swift:${SWIFTVERSION}-${UBUNTUDISTRO}-slim AS runner diff --git a/Scripts/build-release b/Scripts/build-release index cac1c142..178bb62f 100755 --- a/Scripts/build-release +++ b/Scripts/build-release @@ -1,34 +1,22 @@ #!/bin/bash +# (Cross-)compile D2 + set -e cd "$(dirname $0)/.." -# Set up cross-compilation if needed - extra_flags=() if [ -n "$TARGETARCH" ] && [ -n "$TARGETSYSROOT" ]; then echo "==> Setting flags for target arch '$TARGETARCH'" arch_name="$(Scripts/get-linux-arch-name $TARGETARCH)" - # Workaround for https://github.com/stephencelis/CSQLite/pull/1 - if [ ! -f /usr/include/sqlite3.h ]; then - ln -s {"$TARGETSYSROOT",}/usr/include/sqlite3.h - fi - - # There seem to be some strange errors with (glibc) - # on host + target x86_64. Since this header seems to be provided - # elsewhere, we can remove it from the target sysroot as a workaround. - if [ "$BUILDARCH" = amd64 ] && [ "$TARGETARCH" = amd64 ] && [ -f "$TARGETSYSROOT/usr/include/tgmath.h" ]; then - mv "$TARGETSYSROOT/usr/include/tgmath.h{,.backup}" - fi - extra_flags+=( --arch "$arch_name" --sdk "$TARGETSYSROOT" -Xswiftc -use-ld=lld + -Xswiftc -tools-directory -Xswiftc "/usr/$arch_name-linux-gnu/bin" -Xswiftc -resource-dir -Xswiftc "$TARGETSYSROOT/usr/lib/swift" - # TODO: Figure out which of these flags are redundant -Xcc -I"$TARGETSYSROOT/usr/include" -Xcc -I"$TARGETSYSROOT/usr/include/freetype2" -Xcc -I"$TARGETSYSROOT/usr/include/$arch_name-linux-gnu" diff --git a/Scripts/install-build-dependencies-apt b/Scripts/install-build-dependencies-apt index dfa9700a..c5c7105c 100755 --- a/Scripts/install-build-dependencies-apt +++ b/Scripts/install-build-dependencies-apt @@ -6,7 +6,6 @@ set -e cd "$(dirname $0)/.." dependencies=( - build-essential libcairo2-dev libsqlite3-dev libleptonica-dev diff --git a/Scripts/prepare-docker-buildroot b/Scripts/prepare-docker-buildroot new file mode 100755 index 00000000..fee99e97 --- /dev/null +++ b/Scripts/prepare-docker-buildroot @@ -0,0 +1,49 @@ +#!/bin/bash + +# Prepares the build stage during the Docker image build. This assumes +# that the target sysroot has already been installed to $TARGETSYSROOT. + +set -e +cd "$(dirname $0)/.." + +if [ -z "$BUILDARCH" ]; then + echo "Please make sure to specify BUILDARCH!" + exit 1 +fi + +if [ -z "$TARGETARCH" ]; then + echo "Please make sure to specify TARGETARCH!" + exit 1 +fi + +if [ -z "$TARGETSYSROOT" ]; then + echo "Please make sure to specify TARGETSYSROOT!" + exit 1 +fi + +arch_name="$(Scripts/get-linux-arch-name $TARGETARCH)" + +echo "==> Installing cross-GCC" +apt-get update +apt-get install -y gcc-$arch_name-linux-gnu + +# Installing cross-GCC seems to install libstdc++, but not +# link it. Therefore we'll create this manually. +if [ ! -f /usr/$arch_name-linux-gnu/lib/libstdc++.so ]; then + echo "==> Symlinking libstdc++" + ln -s /usr/$arch_name-linux-gnu/lib/libstdc++.so{.6,} +fi + +# Workaround for https://github.com/stephencelis/CSQLite/pull/1 +if [ ! -f /usr/include/sqlite3.h ]; then + echo "==> Patching sqlite3.h" + ln -s {"$TARGETSYSROOT",}/usr/include/sqlite3.h +fi + +# There seem to be some strange errors with (glibc) +# on host + target x86_64. Since this header seems to be provided +# elsewhere, we can remove it from the target sysroot as a workaround. +if [ "$BUILDARCH" = amd64 ] && [ "$TARGETARCH" = amd64 ] && [ -f "$TARGETSYSROOT/usr/include/tgmath.h" ]; then + echo "==> Patching tgmath.h" + mv "$TARGETSYSROOT/usr/include/tgmath.h{,.backup}" +fi From 4abaabe46a7f7060932eafff14a6567dbc95316f Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 16:30:30 +0200 Subject: [PATCH 53/65] Install native GCC if not cross-compiling --- Scripts/prepare-docker-buildroot | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/Scripts/prepare-docker-buildroot b/Scripts/prepare-docker-buildroot index fee99e97..03fb6e09 100755 --- a/Scripts/prepare-docker-buildroot +++ b/Scripts/prepare-docker-buildroot @@ -23,15 +23,20 @@ fi arch_name="$(Scripts/get-linux-arch-name $TARGETARCH)" -echo "==> Installing cross-GCC" +echo "==> Installing build essentials..." apt-get update -apt-get install -y gcc-$arch_name-linux-gnu - -# Installing cross-GCC seems to install libstdc++, but not -# link it. Therefore we'll create this manually. -if [ ! -f /usr/$arch_name-linux-gnu/lib/libstdc++.so ]; then - echo "==> Symlinking libstdc++" - ln -s /usr/$arch_name-linux-gnu/lib/libstdc++.so{.6,} +apt-get install -y build-essential + +if [ "$BUILDARCH" != "$TARGETARCH" ]; then + echo "==> Installing cross-GCC" + apt-get install -y gcc-$arch_name-linux-gnu + + # Installing cross-GCC seems to install libstdc++, but not + # link it. Therefore we'll create this manually. + if [ ! -f /usr/$arch_name-linux-gnu/lib/libstdc++.so ]; then + echo "==> Symlinking libstdc++" + ln -s /usr/$arch_name-linux-gnu/lib/libstdc++.so{.6,} + fi fi # Workaround for https://github.com/stephencelis/CSQLite/pull/1 From ebed54f1e17daf9616b84254afd8e133d30dc9fc Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 16:34:26 +0200 Subject: [PATCH 54/65] Symlink libc into /lib/$arch_name-linux-gnu --- Scripts/prepare-docker-buildroot | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Scripts/prepare-docker-buildroot b/Scripts/prepare-docker-buildroot index 03fb6e09..142fe6b1 100755 --- a/Scripts/prepare-docker-buildroot +++ b/Scripts/prepare-docker-buildroot @@ -37,6 +37,12 @@ if [ "$BUILDARCH" != "$TARGETARCH" ]; then echo "==> Symlinking libstdc++" ln -s /usr/$arch_name-linux-gnu/lib/libstdc++.so{.6,} fi + + # For some reason, libc seems to be expected in /lib/$arch_name-linux-gnu + if [ ! -f /lib/$arch_name-linux-gnu/libc.so ]; then + echo "==> Symlinking libc" + ln -s /usr/$arch_name-linux-gnu/lib/libc.so /lib/$arch_name-linux-gnu/libc.so + fi fi # Workaround for https://github.com/stephencelis/CSQLite/pull/1 From f0d75ed575ff41a11acc4c8a7c0115e4a65cba51 Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 16:42:11 +0200 Subject: [PATCH 55/65] Fix typo --- Scripts/prepare-docker-buildroot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/prepare-docker-buildroot b/Scripts/prepare-docker-buildroot index 142fe6b1..778ed257 100755 --- a/Scripts/prepare-docker-buildroot +++ b/Scripts/prepare-docker-buildroot @@ -56,5 +56,5 @@ fi # elsewhere, we can remove it from the target sysroot as a workaround. if [ "$BUILDARCH" = amd64 ] && [ "$TARGETARCH" = amd64 ] && [ -f "$TARGETSYSROOT/usr/include/tgmath.h" ]; then echo "==> Patching tgmath.h" - mv "$TARGETSYSROOT/usr/include/tgmath.h{,.backup}" + mv "$TARGETSYSROOT/usr/include/tgmath.h"{,.backup} fi From f0f441d5aebcc74ac2c6754f124667eac3dca2ad Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 17:03:19 +0200 Subject: [PATCH 56/65] Install native GCC into target sysroot again --- Scripts/install-build-dependencies-apt | 1 + 1 file changed, 1 insertion(+) diff --git a/Scripts/install-build-dependencies-apt b/Scripts/install-build-dependencies-apt index c5c7105c..dfa9700a 100755 --- a/Scripts/install-build-dependencies-apt +++ b/Scripts/install-build-dependencies-apt @@ -6,6 +6,7 @@ set -e cd "$(dirname $0)/.." dependencies=( + build-essential libcairo2-dev libsqlite3-dev libleptonica-dev From fda56afc901734052f03c6e415d5a28d0716f05c Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 17:04:36 +0200 Subject: [PATCH 57/65] Remove libc workaround again --- Scripts/prepare-docker-buildroot | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Scripts/prepare-docker-buildroot b/Scripts/prepare-docker-buildroot index 778ed257..53cd424f 100755 --- a/Scripts/prepare-docker-buildroot +++ b/Scripts/prepare-docker-buildroot @@ -37,12 +37,6 @@ if [ "$BUILDARCH" != "$TARGETARCH" ]; then echo "==> Symlinking libstdc++" ln -s /usr/$arch_name-linux-gnu/lib/libstdc++.so{.6,} fi - - # For some reason, libc seems to be expected in /lib/$arch_name-linux-gnu - if [ ! -f /lib/$arch_name-linux-gnu/libc.so ]; then - echo "==> Symlinking libc" - ln -s /usr/$arch_name-linux-gnu/lib/libc.so /lib/$arch_name-linux-gnu/libc.so - fi fi # Workaround for https://github.com/stephencelis/CSQLite/pull/1 From 5e09f492fdc1be3befd4a45872d52df6015c6528 Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 17:09:48 +0200 Subject: [PATCH 58/65] Only deploy on main --- .github/workflows/deploy.yml | 1 + .github/workflows/docker.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 73a603c8..a5a56c5e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -11,6 +11,7 @@ on: jobs: deploy: runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' steps: - uses: actions/checkout@v3 - name: Set up Kubernetes CLI diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6dec8016..57959699 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -27,7 +27,7 @@ jobs: with: context: . platforms: linux/amd64,linux/arm64/v8 - push: true + push: true # TODO: Only push on main tags: ghcr.io/fwcd/d2:latest cache-from: type=registry,ref=ghcr.io/fwcd/d2-buildcache:buildcache cache-to: type=registry,ref=ghcr.io/fwcd/d2-buildcache:buildcache,mode=max From abf64e06a698f4b49b12019c28e2b9b324c56e24 Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 17:47:20 +0200 Subject: [PATCH 59/65] Reorder -tools-directory --- Scripts/build-release | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/build-release b/Scripts/build-release index 178bb62f..43b311b4 100755 --- a/Scripts/build-release +++ b/Scripts/build-release @@ -15,8 +15,8 @@ if [ -n "$TARGETARCH" ] && [ -n "$TARGETSYSROOT" ]; then --arch "$arch_name" --sdk "$TARGETSYSROOT" -Xswiftc -use-ld=lld - -Xswiftc -tools-directory -Xswiftc "/usr/$arch_name-linux-gnu/bin" -Xswiftc -resource-dir -Xswiftc "$TARGETSYSROOT/usr/lib/swift" + -Xswiftc -tools-directory -Xswiftc "/usr/$arch_name-linux-gnu/bin" -Xcc -I"$TARGETSYSROOT/usr/include" -Xcc -I"$TARGETSYSROOT/usr/include/freetype2" -Xcc -I"$TARGETSYSROOT/usr/include/$arch_name-linux-gnu" From 36c223ba5879ad4d01e35ab613d02802f8fa2d1d Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 17:48:27 +0200 Subject: [PATCH 60/65] Install native GCC only if not cross-compiling --- Scripts/prepare-docker-buildroot | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Scripts/prepare-docker-buildroot b/Scripts/prepare-docker-buildroot index 53cd424f..ad22874c 100755 --- a/Scripts/prepare-docker-buildroot +++ b/Scripts/prepare-docker-buildroot @@ -23,11 +23,13 @@ fi arch_name="$(Scripts/get-linux-arch-name $TARGETARCH)" -echo "==> Installing build essentials..." +echo "==> Updating apt..." apt-get update -apt-get install -y build-essential -if [ "$BUILDARCH" != "$TARGETARCH" ]; then +if [ "$BUILDARCH" == "$TARGETARCH" ]; then + echo "==> Installing build essentials" + apt-get install -y build-essential +else echo "==> Installing cross-GCC" apt-get install -y gcc-$arch_name-linux-gnu From 486dfe98da5747dd161f8df13e4ccfe01c409ed7 Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 18:31:26 +0200 Subject: [PATCH 61/65] Link libstdc++ from target sysroot rather than cross-GCC --- Scripts/prepare-docker-buildroot | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Scripts/prepare-docker-buildroot b/Scripts/prepare-docker-buildroot index ad22874c..82da9e68 100755 --- a/Scripts/prepare-docker-buildroot +++ b/Scripts/prepare-docker-buildroot @@ -33,11 +33,10 @@ else echo "==> Installing cross-GCC" apt-get install -y gcc-$arch_name-linux-gnu - # Installing cross-GCC seems to install libstdc++, but not - # link it. Therefore we'll create this manually. - if [ ! -f /usr/$arch_name-linux-gnu/lib/libstdc++.so ]; then + # Apparently, we need to symlink libstdc++ manually + if [ ! -f "$TARGETSYSROOT/usr/lib/$arch_name-linux-gnu/libstdc++.so" ]; then echo "==> Symlinking libstdc++" - ln -s /usr/$arch_name-linux-gnu/lib/libstdc++.so{.6,} + ln -s "$TARGETSYSROOT/usr/lib/$arch_name-linux-gnu/libstdc++.so"{.6,} fi fi From 052c44eba2ae06216c5faad656fb7e3f81d42ecf Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 18:54:03 +0200 Subject: [PATCH 62/65] Symlink sysroot libs into cross-GCC libs --- Scripts/prepare-docker-buildroot | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Scripts/prepare-docker-buildroot b/Scripts/prepare-docker-buildroot index 82da9e68..3951a25d 100755 --- a/Scripts/prepare-docker-buildroot +++ b/Scripts/prepare-docker-buildroot @@ -30,14 +30,22 @@ if [ "$BUILDARCH" == "$TARGETARCH" ]; then echo "==> Installing build essentials" apt-get install -y build-essential else - echo "==> Installing cross-GCC" - apt-get install -y gcc-$arch_name-linux-gnu - # Apparently, we need to symlink libstdc++ manually if [ ! -f "$TARGETSYSROOT/usr/lib/$arch_name-linux-gnu/libstdc++.so" ]; then echo "==> Symlinking libstdc++" ln -s "$TARGETSYSROOT/usr/lib/$arch_name-linux-gnu/libstdc++.so"{.6,} fi + + # Symlinking the sysroot libs into the cross-GCC libs + # fixes a very strange linking issue where the linker would + # try resolving libc in /lib/$arch_name-linux-gnu. This also + # needs to happen before installing the cross-GCC package. + echo "==> Symlinking sysroot libs into cross-GCC libs" + mkdir -p "/usr/$arch_name-linux-gnu" + ln -s "$TARGETSYSROOT/usr/lib/$arch_name-linux-gnu" "/usr/$arch_name-linux-gnu/lib" + + echo "==> Installing cross-GCC" + apt-get install -y gcc-$arch_name-linux-gnu fi # Workaround for https://github.com/stephencelis/CSQLite/pull/1 From 5b791297ee234efe72466a8bec8fdbc2f8d3784b Mon Sep 17 00:00:00 2001 From: fwcd Date: Tue, 1 Aug 2023 19:33:58 +0200 Subject: [PATCH 63/65] Link target sysroot in runner image ...to make sure D2 finds the Swift standard libraries at runtime. --- Dockerfile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Dockerfile b/Dockerfile index d102b3ec..6ae289c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,6 +37,11 @@ RUN Scripts/build-release FROM swift:${SWIFTVERSION}-${UBUNTUDISTRO}-slim AS runner +ARG TARGETARCH +ARG UBUNTUDISTRO + +ARG TARGETSYSROOT=/usr/local/${TARGETARCH}-ubuntu-${UBUNTUDISTRO} + # Install Curl, add-apt-repository and node package repository RUN apt-get update && apt-get install -y curl software-properties-common && rm -rf /var/lib/apt/lists/* RUN curl -sL https://deb.nodesource.com/setup_18.x | bash - @@ -45,6 +50,10 @@ RUN curl -sL https://deb.nodesource.com/setup_18.x | bash - COPY Scripts/install-runtime-dependencies-apt Scripts/ RUN Scripts/install-runtime-dependencies-apt && rm -rf /var/lib/apt/lists/* +# Link 'sysroot' to / to make sure D2 can find the Swift stdlibs +# (the runpath within the D2 executable still points to its /usr/lib/swift) +RUN ln -s / ${TARGETSYSROOT} + WORKDIR /opt/d2 # Install Node dependencies From 8bf4abbce5ff811519942349373869086c0dc7de Mon Sep 17 00:00:00 2001 From: FW <30873659+fwcd@users.noreply.github.com> Date: Wed, 2 Aug 2023 01:05:09 +0200 Subject: [PATCH 64/65] Remove native GCC from sysroot again --- Scripts/install-build-dependencies-apt | 1 - 1 file changed, 1 deletion(-) diff --git a/Scripts/install-build-dependencies-apt b/Scripts/install-build-dependencies-apt index dfa9700a..c5c7105c 100755 --- a/Scripts/install-build-dependencies-apt +++ b/Scripts/install-build-dependencies-apt @@ -6,7 +6,6 @@ set -e cd "$(dirname $0)/.." dependencies=( - build-essential libcairo2-dev libsqlite3-dev libleptonica-dev From e59ede96f3996955af1594ab4afe47c1077975bb Mon Sep 17 00:00:00 2001 From: FW <30873659+fwcd@users.noreply.github.com> Date: Wed, 2 Aug 2023 01:42:41 +0200 Subject: [PATCH 65/65] Disable build cache The layers are probably too large for this --- .github/workflows/docker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 57959699..678229bc 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -29,5 +29,4 @@ jobs: platforms: linux/amd64,linux/arm64/v8 push: true # TODO: Only push on main tags: ghcr.io/fwcd/d2:latest - cache-from: type=registry,ref=ghcr.io/fwcd/d2-buildcache:buildcache - cache-to: type=registry,ref=ghcr.io/fwcd/d2-buildcache:buildcache,mode=max +