copy zephyr based sof image toml files from rimage to sof and split to platform and module toml #14747

Workflow file for this run

	---

	name: Zephyr

	# 'workflow_dispatch' allows running this workflow manually from the
	# 'Actions' tab
	# yamllint disable-line rule:truthy
	on: [push, pull_request, workflow_dispatch, workflow_call]

	# Specifies group name that stops previous wokrflows if the name matches
	concurrency:
	# eg. "Zephyr-pull_request-my_fork_branch_to_merge"
	# eg. "Zephyr-push-refs/heads/my_branch_merging"
	group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref \|\| github.ref }}
	cancel-in-progress: true

	jobs:

	manifest-check:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	with:
	path: ./workspace/sof

	- name: plain west update
	run: \|

	: This plain 'west update' does not provide 100% certainty that
	: all the manifest revisions make sense but it is quick and
	: will catch many revision problems. Other jobs typically
	: use 'west update --narrow' which is faster but
	: also able to fetch "wild" SHA1s from any random place! --narrow
	: is useful for testing unmerged Zephyr commits but risks
	: accepting "invalid" ones, this will not.

	pip3 install west
	cd workspace/sof/
	west init -l
	west update --fetch-opt=--filter=tree:0

	# Temporary check until we change west.yml and stop nesting rimage
	# and tomlc99 inside sof which will cleanly separate them from
	# XTOS submodules and... temporarily break every CI, which is why
	# it hasn't been done yet.
	- name: git submodules consistency
	run: \|

	cd workspace/sof
	git submodule update --init --recursive

	west update
	if git status --porcelain=v2 \| grep ^ ; then
	git status
	echo 'FAIL: inconsistency between git submodules and west.yml!'
	echo 'See rimage comment in west.yml.'
	echo 'Always use "git status"'
	exit 1
	fi


	# Temporary hacks to compile the very first LP64 configuration early
	# before it becomes supported by the regular
	# sof/scripts/xtensa-build-zephyr.py configuration script. Then this
	# job will be disappear, folded back in the regular build-* jobs below.
	LP64-WIP:
	runs-on: ubuntu-22.04

	steps:
	- uses: actions/checkout@v3
	with:
	path: ./workspace/sof

	- name: west clones
	run: pip3 install west && cd workspace/sof/ && west init -l &&
	west update --narrow --fetch-opt=--filter=tree:0

	# Not strictly necessary but saves a lot of scrolling in the next step
	# Caching a 12G image is unfortunately not possible:
	# https://github.com/ScribeMD/docker-cache/issues/304
	# For faster builds we would have to pay for some persistent runners.
	- name: Download docker image && ls /opt/toolchains/
	run: cd workspace && ./sof/zephyr/docker-run.sh ls -l /opt/toolchains/

	- name: build with many warnings, work in progress
	run: \|
	# One of the main problems: the old sof-logger is hardcoded to
	# four 32bits parameters.
	cd workspace && ./sof/zephyr/docker-run.sh /bin/sh -c \
	'ln -s /opt/toolchains/zephyr-sdk-* ~/;
	west build --board mimx93_evk_a55 sof/app'


	build-linux:
	runs-on: ubuntu-22.04
	strategy:
	fail-fast: false
	matrix:
	# These keys are kept short because Github's left column is not resizable.
	# Search "zephyr_revision" and see below what they expand to.
	zephyr_revision: [
	mnfst, # special value: don't override sof/west.yml
	zmain, # Zephyr's main branch
	]
	# Using groups to avoid spamming the small results box with too
	# many lines. Pay attention to COMMAS.
	IPC_platforms: [
	# - IPC3 default
	imx8 imx8x imx8m,
	tgl tgl-h, # UNSUPPORTED! Will be removed
	# - IPC4 default
	mtl, lnl,
	# Temporary testbed for Zephyr development.
	tgl tgl-h,
	]
	build_opts: [""]
	# Sparse matrices are complicated, you must read this page slowly:
	# https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
	include:
	# specify one extra -d combination without affecting the main matrix
	- build_opts: -d
	zephyr_revision: mnfst
	IPC_platforms: mtl

	steps:
	- uses: actions/checkout@v3
	# Download a full clone to fix `git describe`, sof_version.h and
	# build reproducibility. sof.git is still small.
	# This is especially useful for daily builds (but not just).
	with:
	fetch-depth: 0
	path: ./workspace/sof

	- name: west clones

	run: pip3 install west && cd workspace/sof/ && west init -l &&
	west update --narrow --fetch-opt=--filter=tree:0

	- name: select zephyr revision
	run: \|
	cd workspace
	if [ 'mnfst' = '${{ matrix.zephyr_revision }}' ]; then
	rem_rev=$(git -C zephyr rev-parse HEAD)
	else
	case '${{ matrix.zephyr_revision }}' in
	'zmain') rem_rev='main' ;;
	*) echo 'Unknown matrix.zephyr_revision'; exit 1 ;;
	esac
	( cd sof/submanifests/
	sed -e "s#=sof_zephyr_revision_override=#${rem_rev}#" \
	sof-ci-jenkins/zephyr-override-template.yml > test-zephyr-main.yml
	)
	west update --narrow --fetch-opt=--filter=tree:0
	fi

	# Because we used git tricks to speed things up, we now have two git
	# problems:
	#
	# 1. git fetch without a refspec fetches zero tag. So `git describe` is
	# different from everyone else. west normally adds `--tags` as a workaround
	# but this is inefficient because it fetches ALL tags; so `west --narrow`
	# does not use --tags.
	#
	# 2. west fetches using the remote URL, not the remote name. So remote
	# branches (if any) are missing from --decorate below
	#
	# => an "empty" and quick fetch _with_ a refspec and the remote name fixes
	# both issues in no time.

	cd zephyr
	git fetch --filter=tree:0 zephyrproject "$rem_rev":_branch_placeholder
	git branch -D _branch_placeholder

	set -x
	west list
	west status
	git log --oneline -n 5 --decorate --graph --no-abbrev-commit

	# Not strictly necessary but saves a lot of scrolling in the next step
	# Caching a 12G image is unfortunately not possible:
	# https://github.com/ScribeMD/docker-cache/issues/304
	# For faster builds we would have to pay for some persistent runners.
	- name: Download docker image && ls /opt/toolchains/
	run: cd workspace && ./sof/zephyr/docker-run.sh ls -l /opt/toolchains/

	# https://github.com/zephyrproject-rtos/docker-image
	# Note: env variables can be passed to the container with
	# -e https_proxy=...
	- name: build
	run: cd workspace && ./sof/zephyr/docker-run.sh
	./sof/zephyr/docker-build.sh --cmake-args=-DEXTRA_CFLAGS=-Werror
	--cmake-args=-DEXTRA_CXXFLAGS=-Werror
	--cmake-args=-DEXTRA_AFLAGS='-Werror -Wa,--fatal-warnings'
	--cmake-args=--warn-uninitialized
	${{ matrix.build_opts }} ${{ matrix.IPC_platforms }}

	- name: Upload build artifacts
	uses: actions/upload-artifact@v3
	if: ${{ matrix.zephyr_revision == 'mnfst' }}
	with:
	name: linux-build ${{ matrix.build_opts }} ${{ matrix.IPC_platforms }}
	path: \|
	${{ github.workspace }}/workspace/build-sof-staging
	${{ github.workspace }}/workspace/**/compile_commands.json

	build-windows:
	runs-on: windows-latest
	strategy:
	fail-fast: false
	matrix:
	# Using groups to avoid spamming the small results box with too
	# many lines. Pay attention to COMMAS.
	platforms: [
	# - IPC3 default
	imx8 imx8x imx8m,
	tgl tgl-h, # UNSUPPORTED! Will be removed
	# - IPC4 default
	mtl,
	# Very few IPC3 platforms support IPC4 too.
	tgl tgl-h,
	]
	build_opts: [""]
	# Sparse matrices are complicated, see comments on Linux matrix above.
	include:
	- build_opts: -d
	platforms: mtl


	steps:
	- uses: actions/checkout@v3
	with:
	fetch-depth: 0
	path: ./workspace/sof

	# Cache artifacts so we do not overload external servers with downloads
	# Remember to change step key if you change the tools so old cache is not restored,
	# or delete cache manually in Github Actions tab
	- name: Cache unzip
	id: cache-unzip
	uses: actions/[email protected]
	with:
	path: unzip-5.51-1-bin.zip
	key: ${{ runner.os }}-cache-unzip-5-51-1

	- name: Cache wget
	id: cache-wget
	uses: actions/[email protected]
	with:
	path: wget-1.11.4-1-bin.zip
	key: ${{ runner.os }}-cache-wget-1-11-4-1

	# Keep this SDK version identical to the one in
	# sof/zephyr/docker-run.sh
	- name: Cache Zephyr SDK 0.16.1
	id: cache-zephyr-sdk
	uses: actions/[email protected]
	with:
	path: zephyr-sdk-0.16.1_windows-x86_64.7z
	key: ${{ runner.os }}-cache-zephyr-sdk-0-16-1

	# Wget is needed by Zephyr SDK setup.cmd installation script
	- name: Download wget
	if: ${{ steps.cache-wget.outputs.cache-hit != 'true' }}
	run: \|
	curl -L -O http://downloads.sourceforge.net/gnuwin32/wget-1.11.4-1-bin.zip

	- name: Download Zephyr SDK 0.16.1
	if: ${{ steps.cache-zephyr-sdk.outputs.cache-hit != 'true' }}
	run: \| # yamllint disable-line rule:line-length
	curl -L -O `
	https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v0.16.1/zephyr-sdk-0.16.1_windows-x86_64.7z

	# Unzips every .zip package to directory matching its name without extension
	- name: Unzip downloaded packages
	run: \|
	7z x .zip -o
	7z x .7z -o

	- name: Add unzip to system PATH
	run: \|
	echo "${{ github.workspace }}/unzip-5.51-1-bin/bin" \| `
	Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append

	- name: Add wget to system PATH
	run: \|
	echo "${{ github.workspace }}/wget-1.11.4-1-bin/bin" \| `
	Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append

	# Install Zephyr SDK - all toolchains including Host Tools
	# and registering CMake package in the registry
	# setup.cmd may not be called in from msys shell as it does not parse
	# forward slash script input arguments correctly.
	- name: Install Zephyr SDK
	run: zephyr-sdk-0.16.1_windows-x86_64/zephyr-sdk-0.16.1/setup.cmd /t all /h /c

	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: '3.8'

	- name: West install
	run: pip3 install west

	- name: West clone
	working-directory: ${{ github.workspace }}/workspace
	# Keep in sync with build-linux above
	run: \|
	west init -l sof
	west update --narrow --fetch-opt=--filter=tree:0

	# Get some tags to fix `git describe`, see build-linux comments above.
	cd zephyr
	$_rev = "$(git rev-parse HEAD)"
	git fetch --filter=tree:0 zephyrproject "${_rev}:_branch_placeholder"
	git branch -D _branch_placeholder


	# Call Setup Python again to save the PIP packages in cache
	- name: Setup Python
	uses: actions/setup-python@v4
	id: cache-python
	with:
	python-version: '3.8'
	cache: 'pip'
	cache-dependency-path: workspace/zephyr/scripts/requirements.txt

	# All requirements will be satisfied if the restored cache matches existing state
	- name: Validate python PIP cache
	working-directory: ${{ github.workspace }}/workspace
	run: pip install -r zephyr/scripts/requirements.txt

	# MSYS2 provides gcc x64_86 toolchain & openssl
	- name: Initialize MSYS2
	uses: msys2/setup-msys2@v2
	with:
	msystem: MSYS
	install: gcc openssl-devel
	path-type: inherit

	- name: Build
	shell: msys2 {0}
	working-directory: ${{ github.workspace }}/workspace
	run: python sof/scripts/xtensa-build-zephyr.py
	--no-interactive
	--cmake-args=-DEXTRA_CFLAGS=-Werror
	--cmake-args=-DEXTRA_CXXFLAGS=-Werror
	--cmake-args=-DEXTRA_AFLAGS='-Werror -Wa,--fatal-warnings'
	--cmake-args=--warn-uninitialized ${{ matrix.build_opts }} ${{ matrix.platforms }}

	- name: Upload build artifacts
	uses: actions/upload-artifact@v3
	with:
	name: windows-build ${{ matrix.build_opts }} ${{ matrix.platforms }}
	path: \|
	${{ github.workspace }}/workspace/build-sof-staging
	${{ github.workspace }}/workspace/**/compile_commands.json


	compare-linux-win:

	runs-on: ubuntu-latest

	# - We don't compare _all_ the builds, and
	# - even when some of the ones we compare fail, we still want to compare the rest.
	if: ${{ always() }}
	needs: [build-linux, build-windows]

	steps:
	- uses: actions/checkout@v3
	# we need only one script but it's simpler to get the (last
	# revision of the) whole repo and it takes seconds.
	with:
	# Isolate the clone in a subdirectory to make sure globbing
	# does not catch random SOF files.
	path: ./sof

	- name: Download Windows and Linux builds
	uses: actions/download-artifact@v3

	- name: apt-get dos2unix
	run: sudo apt-get update; sudo apt-get -y install dos2unix

	- name: Delete and fix expected differences
	run: \|
	ls -l
	# run it twice to make sure it's idempotent
	for i in 0 1; do
	./sof/zephyr/scripts/clean-expected-release-differences.sh \
	windows-build* linux-build*
	done

	- name: Compare Linux vs Windows builds
	run: \|
	diffs=0; ls -l

	# Check not too much is missing (it happened!)
	# http://mywiki.wooledge.org/ParsingLs
	for regdir in 'linux-build *mtl' \
	'windows-build *mtl' \
	'windows-build *tgl tgl-h'; do
	find . -maxdepth 1 \| grep -q "\./${regdir}\$" \|\|
	{ >&2 printf 'Missing %s\n' "${regdir}"; exit 1; }
	done

	set -x
	for windir in windows-build*; do
	lindir=linux-"${windir#windows-}"
	diff -qr "$lindir" "$windir" \|\| : $((diffs++))
	done
	exit $diffs

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

copy zephyr based sof image toml files from rimage to sof and split to platform and module toml #14747

Workflow file

copy zephyr based sof image toml files from rimage to sof and split to platform and module toml #14747

Jobs

Run details

Workflow file for this run