Build

Propagate custom_info Dict through agent Resource #10912

Workflow file for this run

.github/workflows/pythonbuild.yml at 67865b7

	name: Build

	# Schedule runs to run twice a day

	on:
	push:
	branches:
	- master
	pull_request:
	schedule:
	- cron: "0 13 * * *" # This schedule runs at 1pm UTC every day

	env:
	FLYTE_SDK_LOGGING_LEVEL: 10 # debug

	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: true

	jobs:
	detect-python-versions:
	runs-on: ubuntu-latest
	outputs:
	python-versions: ${{ env.python_versions }}
	steps:
	- name: Set Python versions for run
	run: \|
	if [[ ${{ github.event_name }} == "schedule" ]]; then
	echo "python_versions=[\"3.8\",\"3.9\",\"3.10\",\"3.11\",\"3.12\"]" >> $GITHUB_ENV
	else
	echo "python_versions=[\"3.12\"]" >> $GITHUB_ENV
	fi

	build:
	needs:
	- detect-python-versions
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, windows-latest, macos-latest]
	python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
	steps:
	- uses: actions/checkout@v4
	- name: 'Clear action cache'
	uses: ./.github/actions/clear-action-cache
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Cache pip
	uses: actions/cache@v3
	with:
	# This path is specific to Ubuntu
	path: ~/.cache/pip
	# Look to see if there is a cache hit for the corresponding requirements files
	key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
	- name: Install dependencies
	run: \|
	make setup
	pip uninstall -y pandas
	pip freeze
	- name: Test with coverage
	run: \|
	make unit_test_codecov
	- name: Codecov
	uses: codecov/[email protected]
	with:
	fail_ci_if_error: false
	files: coverage.xml

	build-with-extras:
	needs:
	- detect-python-versions
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, windows-latest, macos-latest]
	python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
	steps:
	- uses: actions/checkout@v4
	- name: 'Clear action cache'
	uses: ./.github/actions/clear-action-cache
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Cache pip
	uses: actions/cache@v3
	with:
	# This path is specific to Ubuntu
	path: ~/.cache/pip
	# Look to see if there is a cache hit for the corresponding requirements files
	key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
	- name: Install dependencies
	run: \|
	make setup
	pip uninstall -y pandas
	pip freeze
	- name: Run extras unit tests with coverage
	# Skip this step if running on python 3.12 due to https://github.com/tensorflow/tensorflow/issues/62003
	# and https://github.com/pytorch/pytorch/issues/110436
	if: ${{ matrix.python-version != '3.12' }}
	run: \|
	make unit_test_extras_codecov
	- name: Codecov
	uses: codecov/[email protected]
	with:
	fail_ci_if_error: false
	files: coverage.xml

	build-with-pandas:
	needs:
	- detect-python-versions
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest]
	python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
	pandas: ["pandas<2.0.0", "pandas>=2.0.0"]
	steps:
	- uses: actions/checkout@v4
	- name: 'Clear action cache'
	uses: ./.github/actions/clear-action-cache
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Cache pip
	uses: actions/cache@v3
	with:
	# This path is specific to Ubuntu
	path: ~/.cache/pip
	# Look to see if there is a cache hit for the corresponding requirements files
	key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
	- name: Install dependencies
	run: \|
	make setup
	pip install --force-reinstall "${{ matrix.pandas }}"
	pip freeze
	- name: Test with coverage
	run: \|
	make unit_test_codecov
	- name: Codecov
	uses: codecov/[email protected]
	with:
	fail_ci_if_error: false
	files: coverage.xml

	test-hypothesis:
	needs:
	- detect-python-versions
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest]
	python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
	steps:
	- uses: actions/checkout@v4
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Cache pip
	uses: actions/cache@v3
	with:
	# This path is specific to Ubuntu
	path: ~/.cache/pip
	# Look to see if there is a cache hit for the corresponding requirements files
	key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
	- name: Install dependencies
	run: make setup && pip freeze
	- name: Test with coverage
	env:
	FLYTEKIT_HYPOTHESIS_PROFILE: ci
	run: \|
	make unit_test_hypothesis
	- name: Codecov
	uses: codecov/[email protected]
	with:
	fail_ci_if_error: false
	files: coverage.xml

	test-serialization:
	needs:
	- detect-python-versions
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest]
	python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
	steps:
	- uses: actions/checkout@v4
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Cache pip
	uses: actions/cache@v3
	with:
	# This path is specific to Ubuntu
	path: ~/.cache/pip
	# Look to see if there is a cache hit for the corresponding requirements files
	key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
	- name: Install dependencies
	run: make setup && pip freeze
	- name: Test with coverage
	run: \|
	make test_serialization_codecov
	- name: Codecov
	uses: codecov/[email protected]
	with:
	fail_ci_if_error: false
	files: coverage.xml

	integration:
	needs:
	- detect-python-versions
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest]
	python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
	steps:
	# As described in https://github.com/pypa/setuptools_scm/issues/414, SCM needs git history
	# and tags to work.
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Cache pip
	uses: actions/cache@v3
	with:
	# This path is specific to Ubuntu
	path: ~/.cache/pip
	# Look to see if there is a cache hit for the corresponding requirements files
	key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
	- name: Install dependencies
	run: make setup && pip freeze
	- name: Install FlyteCTL
	uses: unionai-oss/flytectl-setup-action@master
	- name: Setup Flyte Sandbox
	run: \|
	flytectl demo start
	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3
	with:
	driver-opts: \|
	image=moby/buildkit:master
	network=host
	- name: Build and push to local registry
	uses: docker/build-push-action@v5
	with:
	context: .
	file: Dockerfile.dev
	build-args: \|
	PYTHON_VERSION=${{ matrix.python-version }}
	push: true
	tags: localhost:30000/flytekit:dev
	cache-from: type=gha
	cache-to: type=gha,mode=max
	- name: Integration Test with coverage
	env:
	FLYTEKIT_IMAGE: localhost:30000/flytekit:dev
	FLYTEKIT_CI: 1
	PYTEST_OPTS: -n2
	run: make integration_test_codecov
	- name: Codecov
	uses: codecov/[email protected]
	with:
	fail_ci_if_error: false

	build-plugins:
	needs:
	- build
	- detect-python-versions
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
	plugin-names:
	# Please maintain an alphabetical order in the following list
	- flytekit-airflow
	- flytekit-async-fsspec
	- flytekit-aws-athena
	- flytekit-aws-batch
	- flytekit-aws-sagemaker
	- flytekit-bigquery
	- flytekit-dask
	- flytekit-data-fsspec
	- flytekit-dbt
	- flytekit-deck-standard
	- flytekit-dolt
	- flytekit-duckdb
	- flytekit-envd
	- flytekit-flyteinteractive
	- flytekit-greatexpectations
	- flytekit-hive
	- flytekit-huggingface
	- flytekit-identity-aware-proxy
	- flytekit-k8s-pod
	- flytekit-kf-mpi
	- flytekit-kf-pytorch
	- flytekit-kf-tensorflow
	- flytekit-mlflow
	- flytekit-mmcloud
	- flytekit-modin
	- flytekit-onnx-pytorch
	- flytekit-onnx-scikitlearn
	# onnx-tensorflow needs a version of tensorflow that does not work with protobuf>4.
	# The issue is being tracked on the tensorflow side in https://github.com/tensorflow/tensorflow/issues/53234#issuecomment-1330111693
	# flytekit-onnx-tensorflow
	- flytekit-openai
	- flytekit-pandera
	- flytekit-papermill
	- flytekit-polars
	- flytekit-pydantic
	- flytekit-ray
	- flytekit-snowflake
	- flytekit-spark
	- flytekit-sqlalchemy
	- flytekit-vaex
	- flytekit-whylogs
	exclude:
	# flytekit-modin depends on ray which does not have a 3.11 wheel yet.
	# Issue tracked in https://github.com/ray-project/ray/issues/27881
	- python-version: 3.11
	plugin-names: "flytekit-modin"
	- python-version: 3.11
	plugin-names: "flytekit-ray"
	# Great-expectations does not support python 3.11 due to sqlachemy>=2.0.0
	# not being supported yet:
	# https://github.com/great-expectations/great_expectations/issues/7020
	- python-version: 3.11
	plugin-names: "flytekit-greatexpectations"
	# onnxruntime does not support python 3.10 yet
	# https://github.com/microsoft/onnxruntime/issues/9782
	- python-version: 3.11
	plugin-names: "flytekit-onnx-pytorch"
	- python-version: 3.11
	plugin-names: "flytekit-onnx-scikitlearn"
	- python-version: 3.11
	plugin-names: "flytekit-onnx-tensorflow"
	# vaex currently doesn't support python 3.11
	- python-version: 3.11
	plugin-names: "flytekit-vaex"
	# whylogs does support python 3.11 dataclass restrictions
	# See: https://github.com/flyteorg/flytekit/actions/runs/4493746408/jobs/7905368664
	- python-version: 3.11
	plugin-names: "flytekit-whylogs"
	# apache-beam, one of flytekit-airflow dependencies, does not support python 3.12: https://github.com/apache/beam/issues/29149
	- python-version: 3.12
	plugin-names: "flytekit-airflow"
	# ydata-profiling, a dependency of flytekit-deck-standard, does not support python 3.12: https://github.com/ydataai/ydata-profiling/issues/1510
	- python-version: 3.12
	plugin-names: "flytekit-deck-standard"
	# Tensorflow is a dependency of flytekit-mlflow tests and that is not supported yet: https://github.com/tensorflow/tensorflow/issues/62003
	- python-version: 3.12
	plugin-names: "flytekit-mlflow"
	# modin[ray] is a dependency needed to run the tests, unfortunately this is not supported in python 3.12 yet
	- python-version: 3.12
	plugin-names: "flytekit-modin"
	# vaex currently doesn't support python 3.12
	- python-version: 3.12
	plugin-names: "flytekit-vaex"
	# Ray does not support python 3.12 yet: https://github.com/ray-project/ray/issues/40211
	- python-version: 3.12
	plugin-names: "flytekit-ray"
	# Segmentation fault on python 3.12: https://github.com/flyteorg/flyte/issues/5020
	- python-version: 3.12
	plugin-names: "flytekit-kf-pytorch"
	steps:
	- uses: actions/checkout@v4
	- name: 'Clear action cache'
	uses: ./.github/actions/clear-action-cache
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Cache pip
	uses: actions/cache@v3
	with:
	# This path is specific to Ubuntu
	path: ~/.cache/pip
	# Look to see if there is a cache hit for the corresponding requirements files
	key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('plugins/{0}/requirements.txt', matrix.plugin-names ))) }}
	- name: Install dependencies
	run: \|
	# TODO: double-check if checking out all tags solves the issue
	export SETUPTOOLS_SCM_PRETEND_VERSION="3.0.0"
	make setup
	cd plugins/${{ matrix.plugin-names }}
	pip install .
	if [ -f dev-requirements.in ]; then pip install -r dev-requirements.in; fi
	pip install -U $GITHUB_WORKSPACE
	pip freeze
	- name: Test with coverage
	run: \|
	cd plugins/${{ matrix.plugin-names }}
	# onnx plugins does not support protobuf>4 yet (in fact it is tensorflow that
	# does not support that yet). More details in https://github.com/onnx/onnx/issues/4239.
	if [[ ${{ matrix.plugin-names }} == "onnx" \|\| ${{ matrix.plugin-names }} == "flytekit-whylogs" \|\| ${{ matrix.plugin-names }} == "flytekit-mlflow" ]]; then
	PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python coverage run -m pytest tests --cov=./ --cov-report=xml --cov-append
	else
	coverage run -m pytest tests --cov=./ --cov-report=xml --cov-append
	fi
	- name: Codecov
	uses: codecov/[email protected]
	with:
	fail_ci_if_error: false
	lint:
	runs-on: ubuntu-latest
	steps:
	- name: Fetch the code
	uses: actions/checkout@v4
	- name: Set up Python 3.12
	uses: actions/setup-python@v4
	with:
	python-version: 3.12
	- uses: actions/cache@v3
	with:
	path: ~/.cache/pip
	key: ${{ runner.os }}-pip-${{ hashFiles('**/dev-requirements.in') }}
	restore-keys: \|
	${{ runner.os }}-pip-
	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	make setup
	pip freeze
	- name: Lint
	run: \|
	make lint
	- name: ShellCheck
	uses: ludeeus/action-shellcheck@master
	with:
	ignore_paths: boilerplate

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Propagate custom_info Dict through agent Resource #10912

Workflow file

Propagate custom_info Dict through agent Resource #10912

Jobs

Run details

Workflow file for this run