Skip to content

Build ffi libraries #58

Build ffi libraries

Build ffi libraries #58

Workflow file for this run

---
name: Build ffi libraries
'on':
push:
tags:
- 'v[0-9]+.[0-9]+.[0-9]*'
branches:
- 'build-all-*'
- 'build-ffis-*'
schedule:
- cron: '05 00 * * *'
workflow_dispatch:
env:
CARGO_UNSTABLE_SPARSE_REGISTRY: true
CARGO: cargo
# CARGO_OPTIONS: "--verbose"
CARGO_OPTIONS: "--release"
TARI_NETWORK_CHANGELOG: 'development'
TOOLCHAIN: 'stable'
## Must be a JSon string
BUILD_LIBFFIS: '["minotari_wallet_ffi","minotari_chat_ffi"]'
# BUILD_LIBFFIS: '["minotari_chat_ffi"]'
matrix-json-file: '.github/workflows/build_libffis.json'
concurrency:
# https://docs.github.com/en/actions/examples/using-concurrency-expressions-and-a-test-matrix
group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/v') || github.ref != 'refs/heads/development' || github.ref != 'refs/heads/nextnet' || github.ref != 'refs/heads/stagenet' }}
permissions: {}
jobs:
matrix-prep:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
libffiss: ${{ steps.set-matrix.outputs.libffiss }}
steps:
- name: Checkout source code
uses: actions/checkout@v4
- name: Set Matrix from JSon file
id: set-matrix
run: |
## build all targets images
# matrix=$( jq -s -c .[] ${{ env.matrix-json-file }} )
#
## build only single target image
# matrix_selection=$( jq -c '.[] | select( ."runs-on" == "ubuntu-latest" )' ${{ env.matrix-json-file }} )
# matrix_selection=$( jq -c '.[] | select( ."target" == "x86_64-linux-android" )' ${{ env.matrix-json-file }} )
# matrix_selection=$( jq -c '.[] | select( ."target" | contains("android") )' ${{ env.matrix-json-file }} )
#
## buid select target images - build_enabled
matrix_selection=$( jq -c '.[] | select( ."build_enabled" != false )' ${{ env.matrix-json-file }} )
#
## Setup json array
matrix=$(echo ${matrix_selection} | jq -s -c)
#
## Setup the json build matrix
# matrix=$(echo ${matrix_selection} | jq -s -c '{"builds": .}')
#
echo $matrix
echo $matrix | jq .
echo "matrix=${matrix}" >> $GITHUB_OUTPUT
echo "libffiss=${BUILD_LIBFFIS}" >> $GITHUB_OUTPUT
matrix-check:
# Debug matrix
if: ${{ false }}
runs-on: ubuntu-latest
needs: matrix-prep
steps:
- name: Install json2yaml
run: |
sudo npm install -g json2yaml
- name: Check matrix definition
run: |
matrix='${{ needs.matrix-prep.outputs.matrix }}'
echo $matrix
echo $matrix | jq .
echo $matrix | json2yaml
libffiss='${{ needs.matrix-prep.outputs.libffiss }}'
echo $libffiss
echo $libffiss | jq .
echo $libffiss | json2yaml
builds:
needs: matrix-prep
strategy:
fail-fast: false
matrix:
libffis: ${{ fromJson(needs.matrix-prep.outputs.libffiss) }}
builds: ${{ fromJson(needs.matrix-prep.outputs.matrix) }}
runs-on: ${{ matrix.builds.runs-on }}
steps:
- name: Checkout source code
uses: actions/checkout@v4
- name: Declare TestNet for tags
# Don't forget to comment out the below if, when force testing with GHA_NETWORK
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
env:
GHA_NETWORK: ${{ github.ref_name }}
# GHA_NETWORK: "v1.0.0-rc.4"
shell: bash
run: |
source buildtools/multinet_envs.sh ${{ env.GHA_NETWORK }}
echo ${TARI_NETWORK}
echo ${TARI_TARGET_NETWORK}
echo ${TARI_NETWORK_CHANGELOG}
echo "TARI_NETWORK=${TARI_NETWORK}" >> $GITHUB_ENV
echo "TARI_TARGET_NETWORK=${TARI_TARGET_NETWORK}" >> $GITHUB_ENV
echo "TARI_NETWORK_CHANGELOG=${TARI_NETWORK_CHANGELOG}" >> $GITHUB_ENV
- name: Declare Android/iOS envs
shell: bash
run: |
if [[ "${{ matrix.builds.target }}" =~ "android" ]]; then
echo "CFLAGS=-DMDB_USE_ROBUST=0" >> $GITHUB_ENV
echo "TARGET_PLATFORM=android" >> $GITHUB_ENV
elif [[ "${{ matrix.builds.target }}" =~ "ios" ]]; then
echo "TARGET_PLATFORM=ios" >> $GITHUB_ENV
if [[ "${{ matrix.builds.target }}" =~ "-sim" ]]; then
echo "TARGET_SIM=-sim" >> $GITHUB_ENV
fi
elif [[ "${{ matrix.builds.target }}" =~ "linux-gnu" ]]; then
echo "TARGET_PLATFORM=linux" >> $GITHUB_ENV
elif [[ "${{ matrix.builds.target }}" =~ "apple-darwin" ]]; then
echo "TARGET_PLATFORM=macos" >> $GITHUB_ENV
fi
# Strip begining
tempEnv="${{ matrix.builds.target }}"
echo "TARGET_ARCH=${tempEnv/-*/}" >> $GITHUB_ENV
# Strip outside of *_X_*
tempEnv="${{ matrix.libffis }}"
tempEnv=${tempEnv#*_}
echo "TARGET_NAME=${tempEnv%_*}" >> $GITHUB_ENV
- name: Install Linux dependencies - Ubuntu
if: ${{ startsWith(runner.os,'Linux') && ( ! matrix.builds.cross ) }}
run: |
sudo apt-get update
sudo bash scripts/install_ubuntu_dependencies.sh
- name: Install macOS dependencies
if: startsWith(runner.os,'macOS')
run: |
brew install openssl cmake coreutils automake autoconf protobuf
rustup target add ${{ matrix.builds.target }}
- name: Setup Rust toolchain
if: ${{ ! matrix.builds.cross }}
uses: dtolnay/rust-toolchain@master
with:
components: rustfmt, clippy
toolchain: ${{ env.TOOLCHAIN }}
targets: ${{ matrix.builds.target }}
- name: Cache cargo files and outputs
if: ${{ ( ! startsWith(github.ref, 'refs/tags/v') ) && ( ! matrix.builds.cross ) }}
uses: Swatinem/rust-cache@v2
with:
key: ${{ matrix.builds.target }}
- name: Install and setup cargo cross
if: ${{ matrix.builds.cross }}
shell: bash
run: |
cargo install cross
echo "CARGO=cross" >> $GITHUB_ENV
- name: Install rust target/toolchain for native/local cross-compile builds
if: ${{ ! matrix.builds.cross }}
# if: ${{ ( ! matrix.builds.cross ) && env.TARGET_PLATFORM != 'ios' }}
run: |
rustup target add ${{ matrix.builds.target }}
# rustup toolchain install stable-${{ matrix.builds.target }} --force-non-host
rustup target list
rustup toolchain list
rustup show
- name: Show command used for Cargo
run: |
echo "cargo command is: ${{ env.CARGO }}"
echo "cargo options is: ${{ env.CARGO_OPTIONS }}"
echo "cross flag: ${{ matrix.builds.cross }}"
- name: Build ${{ matrix.libffis }} libraries
# can't use bash for windows build, perl not supported (openssl)
# don't mess with indentation - breaks inline multiline (readiblity)
run: >
${{ env.CARGO }} build --lib ${{ env.CARGO_OPTIONS }}
--package ${{ matrix.libffis }}
--target ${{ matrix.builds.target }}
--locked
- name: Prepare ${{ matrix.libffis }} libraries
shell: bash
run: |
mkdir -p "${{ runner.temp }}/lib${{ matrix.libffis }}-${{ env.TARGET_PLATFORM }}-${{ env.TARGET_ARCH }}${{ env.TARGET_SIM }}"
cd "${{ runner.temp }}/lib${{ matrix.libffis }}-${{ env.TARGET_PLATFORM }}-${{ env.TARGET_ARCH }}${{ env.TARGET_SIM }}"
cp -v "$GITHUB_WORKSPACE/target/${{ matrix.builds.target }}/release/lib${{ matrix.libffis }}.a" "lib${{ matrix.libffis }}.${{ env.TARGET_PLATFORM }}_${{ env.TARGET_ARCH }}${{ env.TARGET_SIM }}.a"
cp -v "$GITHUB_WORKSPACE/base_layer/${{ env.TARGET_NAME }}_ffi/${{ env.TARGET_NAME }}.h" lib${{ matrix.libffis }}.h
if [ -f "$GITHUB_WORKSPACE/changelog-${{ env.TARI_NETWORK_CHANGELOG }}.md" ]; then
cp -v "$GITHUB_WORKSPACE/changelog-${{ env.TARI_NETWORK_CHANGELOG }}.md" .
TARI_NETWORK_CHANGELOG_FILE=lib${{ matrix.libffis }}-${{ env.TARGET_PLATFORM }}-${{ env.TARGET_ARCH }}${{ env.TARGET_SIM }}/changelog-${{ env.TARI_NETWORK_CHANGELOG }}.md
echo ${TARI_NETWORK_CHANGELOG_FILE}
fi
cd ..
shasum -a 256 \
"lib${{ matrix.libffis }}-${{ env.TARGET_PLATFORM }}-${{ env.TARGET_ARCH }}${{ env.TARGET_SIM }}/lib${{ matrix.libffis }}.${{ env.TARGET_PLATFORM }}_${{ env.TARGET_ARCH }}${{ env.TARGET_SIM }}.a" \
"lib${{ matrix.libffis }}-${{ env.TARGET_PLATFORM }}-${{ env.TARGET_ARCH }}${{ env.TARGET_SIM }}/lib${{ matrix.libffis }}.h" \
"${TARI_NETWORK_CHANGELOG_FILE}" \
> "lib${{ matrix.libffis }}-${{ env.TARGET_PLATFORM }}-${{ env.TARGET_ARCH }}${{ env.TARGET_SIM }}/lib${{ matrix.libffis }}.${{ env.TARGET_PLATFORM }}_${{ env.TARGET_ARCH }}${{ env.TARGET_SIM }}.sha256"
ls -alhtR "${{ runner.temp }}/lib${{ matrix.libffis }}-${{ env.TARGET_PLATFORM }}-${{ env.TARGET_ARCH }}${{ env.TARGET_SIM }}"
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: lib${{ matrix.libffis }}-${{ env.TARGET_PLATFORM }}-${{ env.TARGET_ARCH }}${{ env.TARGET_SIM }}
path: ${{ runner.temp }}/lib${{ matrix.libffis }}-${{ env.TARGET_PLATFORM }}-${{ env.TARGET_ARCH }}${{ env.TARGET_SIM }}
ios_assemble:
name: Assemble iOS multiArch for ${{ matrix.libffis }}
# Disable iOS Assembly workflow
#if: ${{ false }}
#continue-on-error: true
# Limits to only iOS builds?
runs-on: macos-latest
needs: [matrix-prep, builds]
strategy:
fail-fast: false
matrix:
libffis: ${{ fromJson(needs.matrix-prep.outputs.libffiss) }}
steps:
- name: Install macOS dependencies
run: brew install coreutils
- name: Download iOS libffiss for ${{ matrix.libffis }}
uses: actions/download-artifact@v4
with:
# wildcard downloads not supported yet ( minotari_*_ffi-ios-* )
# name: ${{ matrix.libffis }}-ios
path: libffiss
- name: Verify checksums
shell: bash
working-directory: libffiss
run: |
ls -alhtR
find . -name "*.sha256" -type f -print | xargs cat >> libffiss.txt.sha256-verify
cat libffiss.txt.sha256-verify
sha256sum -c libffiss.txt.sha256-verify
- name: Assemble iOS universal libffis
shell: bash
working-directory: libffiss
run: |
ls -alhtR
mkdir lib${{ matrix.libffis }}-ios-universal
cp -v "lib${{ matrix.libffis }}-ios-x86_64/lib${{ matrix.libffis }}.h" \
lib${{ matrix.libffis }}-ios-universal/
echo "Check for changelog"
if [ -f lib${{ matrix.libffis }}-ios-x86_64/changelog-*.md ]; then
echo "Changelog found"
envChangelogFull=$(ls lib${{ matrix.libffis }}-ios-x86_64/changelog-*.md)
echo ${envChangelogFull}
# Strip suffix
#envChangelog=${envChangelogFull::-3}
envChangelog=${envChangelogFull:0:${#envChangelogFull}-3}
echo ${envChangelog}
# Strip prefix
TARI_NETWORK_CHANGELOG=${envChangelog##*/changelog-}
echo ${TARI_NETWORK_CHANGELOG}
cp -v "lib${{ matrix.libffis }}-ios-x86_64/changelog-${TARI_NETWORK_CHANGELOG}.md" \
lib${{ matrix.libffis }}-ios-universal/
TARI_NETWORK_CHANGELOG_FILE=lib${{ matrix.libffis }}-ios-universal/changelog-${TARI_NETWORK_CHANGELOG}.md
echo ${TARI_NETWORK_CHANGELOG_FILE}
else
echo "No changelog found"
fi
lipo -create \
"lib${{ matrix.libffis }}-ios-x86_64/lib${{ matrix.libffis }}.ios_x86_64.a" \
"lib${{ matrix.libffis }}-ios-aarch64/lib${{ matrix.libffis }}.ios_aarch64.a" \
-output "lib${{ matrix.libffis }}-ios-universal/lib${{ matrix.libffis }}.ios_universal.a"
shasum -a 256 \
"lib${{ matrix.libffis }}-ios-universal/lib${{ matrix.libffis }}.ios_universal.a" \
"lib${{ matrix.libffis }}-ios-universal/lib${{ matrix.libffis }}.h" "${TARI_NETWORK_CHANGELOG_FILE}" \
> "lib${{ matrix.libffis }}-ios-universal/lib${{ matrix.libffis }}.ios_universal.sha256"
ls -alhtR
- name: Upload iOS universal libffis artifacts
uses: actions/upload-artifact@v4
with:
name: lib${{ matrix.libffis }}-ios-universal
path: libffiss/lib${{ matrix.libffis }}-ios-universal
- name: Assemble iOS libffis xcframework
shell: bash
working-directory: libffiss
run: |
ls -alhtR
mkdir lib${{ matrix.libffis }}-ios-universal-sim
lipo -create \
"lib${{ matrix.libffis }}-ios-x86_64/lib${{ matrix.libffis }}.ios_x86_64.a" \
"lib${{ matrix.libffis }}-ios-aarch64-sim/lib${{ matrix.libffis }}.ios_aarch64-sim.a" \
-output "lib${{ matrix.libffis }}-ios-universal-sim/lib${{ matrix.libffis }}.ios_universal-sim.a"
mkdir lib${{ matrix.libffis }}-ios-xcframework
echo "Check for changelog"
if [ -f lib${{ matrix.libffis }}-ios-x86_64/changelog-*.md ]; then
echo "Changelog found"
envChangelogFull=$(ls lib${{ matrix.libffis }}-ios-x86_64/changelog-*.md)
echo ${envChangelogFull}
# Strip suffix
#envChangelog=${envChangelogFull::-3}
envChangelog=${envChangelogFull:0:${#envChangelogFull}-3}
echo ${envChangelog}
# Strip prefix
TARI_NETWORK_CHANGELOG=${envChangelog##*/changelog-}
echo ${TARI_NETWORK_CHANGELOG}
cp -v "lib${{ matrix.libffis }}-ios-x86_64/changelog-${TARI_NETWORK_CHANGELOG}.md" \
lib${{ matrix.libffis }}-ios-xcframework/
TARI_NETWORK_CHANGELOG_FILE=lib${{ matrix.libffis }}-ios-xcframework/changelog-${TARI_NETWORK_CHANGELOG}.md
echo ${TARI_NETWORK_CHANGELOG_FILE}
else
echo "No changelog found"
fi
xcodebuild -create-xcframework \
-library "lib${{ matrix.libffis }}-ios-universal-sim/lib${{ matrix.libffis }}.ios_universal-sim.a" \
-headers "lib${{ matrix.libffis }}-ios-x86_64/lib${{ matrix.libffis }}.h" \
-library "lib${{ matrix.libffis }}-ios-aarch64/lib${{ matrix.libffis }}.ios_aarch64.a" \
-headers "lib${{ matrix.libffis }}-ios-aarch64/lib${{ matrix.libffis }}.h" \
-output lib${{ matrix.libffis }}-ios-xcframework/lib${{ matrix.libffis }}_ios.xcframework
shasum -a 256 \
"lib${{ matrix.libffis }}-ios-xcframework/lib${{ matrix.libffis }}_ios.xcframework/Info.plist" \
"lib${{ matrix.libffis }}-ios-xcframework/lib${{ matrix.libffis }}_ios.xcframework/ios-arm64/Headers" \
"lib${{ matrix.libffis }}-ios-xcframework/lib${{ matrix.libffis }}_ios.xcframework/ios-arm64/lib${{ matrix.libffis }}.ios_aarch64.a" \
"lib${{ matrix.libffis }}-ios-xcframework/lib${{ matrix.libffis }}_ios.xcframework/ios-arm64_x86_64-simulator/Headers" \
"lib${{ matrix.libffis }}-ios-xcframework/lib${{ matrix.libffis }}_ios.xcframework/ios-arm64_x86_64-simulator/lib${{ matrix.libffis }}.ios_universal-sim.a" \
"${TARI_NETWORK_CHANGELOG_FILE}" \
> "lib${{ matrix.libffis }}-ios-xcframework/lib${{ matrix.libffis }}.ios_xcframework.sha256"
ls -alhtR
- name: Upload iOS xcframework libffis artifacts
uses: actions/upload-artifact@v4
with:
name: lib${{ matrix.libffis }}-ios-xcframework
path: libffiss/lib${{ matrix.libffis }}-ios-xcframework
create_release:
name: Create release for ffi libraries
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
runs-on: ubuntu-latest
needs: [matrix-prep, builds, ios_assemble]
steps:
- name: Download all ffi libraries
uses: actions/download-artifact@v4
with:
path: libffiss
- name: Verify checksums
shell: bash
working-directory: libffiss
run: |
ls -alhtR
find . -name "*.sha256" -type f -print | xargs cat >> libffiss.txt.sha256-verify
cat libffiss.txt.sha256-verify
sha256sum -c libffiss.txt.sha256-verify
rm -fv ibffiss.txt.sha256-verify
- name: Archive iOS xcframework(s)
shell: bash
working-directory: libffiss
run: |
# Spaces are important - string to array
FFI_DIRS=( $(echo ${BUILD_LIBFFIS} | jq -r '.[]') )
mkdir -p releases
for FFI_DIR in "${FFI_DIRS[@]}"; do
echo "xcframework for ${FFI_DIR}"
if [ -d lib${FFI_DIR}-ios-xcframework ]; then
7z a lib${FFI_DIR}.ios-xcframework.zip lib${FFI_DIR}-ios-xcframework/*
rm -fvr lib${FFI_DIR}-ios-xcframework
shasum -a 256 \
"lib${FFI_DIR}.ios-xcframework.zip" \
> "lib${FFI_DIR}.ios-xcframework.zip.sha256"
mv -v lib${FFI_DIR}.ios-xcframework.zip* releases
fi
done
- name: Release preparation from assets
shell: bash
working-directory: libffiss
run: |
find . -regextype posix-extended -regex '.*\.(a|h|md|sha256)' -type f \
-not \( -path "./releases" -prune \) \
-exec cp -fv {} ./releases/ \;
ls -alhtR
- name: Create release
uses: ncipollo/release-action@v1
with:
artifacts: "libffiss/releases/*"
token: ${{ secrets.GITHUB_TOKEN }}
prerelease: true
draft: true
allowUpdates: true
updateOnlyUnreleased: true
replacesArtifacts: true