-
Notifications
You must be signed in to change notification settings - Fork 670
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
1,202 changed files
with
5,536 additions
and
5,156 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,161 @@ | ||
name: Components Checks | ||
|
||
on: | ||
pull_request: | ||
push: | ||
branches: | ||
- master | ||
env: | ||
GO_VERSION: "1.19" | ||
PRIORITIES: "P0" | ||
jobs: | ||
unpack-envvars: | ||
runs-on: ubuntu-latest | ||
outputs: | ||
go-version: ${{ steps.step.outputs.go-version }} | ||
steps: | ||
- id: step | ||
run: | | ||
echo "go-version=${{ env.GO_VERSION }}" >> $GITHUB_OUTPUT | ||
lint: | ||
name: Lint | ||
needs: | ||
- unpack-envvars | ||
strategy: | ||
fail-fast: false | ||
matrix: | ||
component: | ||
- datacatalog | ||
- flyteadmin | ||
# TODO(monorepo): Enable lint flytecopilot | ||
# - flytecopilot | ||
- flyteidl | ||
- flyteplugins | ||
- flytepropeller | ||
- flytestdlib | ||
uses: ./.github/workflows/lint.yml | ||
with: | ||
component: ${{ matrix.component }} | ||
go-version: ${{ needs.unpack-envvars.outputs.go-version }} | ||
unit-tests: | ||
name: Unit Tests | ||
needs: | ||
- unpack-envvars | ||
strategy: | ||
fail-fast: false | ||
matrix: | ||
component: | ||
- datacatalog | ||
- flyteadmin | ||
- flytecopilot | ||
# TODO(monorepo): Enable flyteidl unit tests | ||
# - flyteidl | ||
- flyteplugins | ||
- flytepropeller | ||
- flytestdlib | ||
uses: ./.github/workflows/unit-tests.yml | ||
with: | ||
component: ${{ matrix.component }} | ||
go-version: ${{ needs.unpack-envvars.outputs.go-version }} | ||
secrets: | ||
FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} | ||
docker-build: | ||
strategy: | ||
fail-fast: false | ||
matrix: | ||
component: | ||
- datacatalog | ||
- flyteadmin | ||
- flytecopilot | ||
- flytepropeller | ||
name: Docker Build Images | ||
uses: ./.github/workflows/component_docker_build.yml | ||
with: | ||
component: ${{ matrix.component }} | ||
|
||
# TODO(monorepo): these tests are broken. They never test an actual change. | ||
# endtoend: | ||
# name: End2End Test | ||
# needs: [ docker-build ] | ||
# uses: ./.github/workflows/end2end.yml | ||
# with: | ||
# # Reusing the output of the matrix is ok as they are essentially writing the same value (i.e. the directory artifacts are written + run id) | ||
# cache_key: ${{ needs.docker-build.outputs.cache_key }} | ||
# priorities: "P0" | ||
|
||
integration: | ||
name: Integration Test | ||
needs: | ||
- docker-build | ||
- unpack-envvars | ||
strategy: | ||
fail-fast: false | ||
matrix: | ||
component: | ||
- flyteadmin | ||
uses: ./.github/workflows/integration.yml | ||
with: | ||
component: ${{ matrix.component }} | ||
cache_key: ${{ needs.docker-build.outputs.cache_key }} | ||
go-version: ${{ needs.unpack-envvars.outputs.go-version }} | ||
|
||
generate: | ||
name: Check Go Generate | ||
needs: | ||
- unpack-envvars | ||
strategy: | ||
fail-fast: false | ||
matrix: | ||
component: | ||
- datacatalog | ||
- flyteadmin | ||
- flytecopilot | ||
- flytepropeller | ||
uses: ./.github/workflows/go_generate.yml | ||
with: | ||
component: ${{ matrix.component }} | ||
go-version: ${{ needs.unpack-envvars.outputs.go-version }} | ||
|
||
# TODO(monorepo): enable bump_version | ||
# bump_version: | ||
# name: Bump Version | ||
# if: ${{ github.event_name != 'pull_request' }} | ||
# needs: [ endtoend, integration, lint, tests, generate ] # Only to ensure it can successfully build | ||
# uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master | ||
# secrets: | ||
# FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} | ||
|
||
# goreleaser: | ||
# name: Goreleaser | ||
# needs: [ bump_version ] # Only to ensure it can successfully build | ||
# uses: flyteorg/flytetools/.github/workflows/goreleaser.yml@master | ||
# with: | ||
# go-version: "1.19" | ||
# secrets: | ||
# FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} | ||
|
||
# push_docker_image: | ||
# name: Build & Push Image | ||
# # TODO(monorepo): depend on bump_version | ||
# # needs: [ bump_version ] | ||
# strategy: | ||
# fail-fast: false | ||
# matrix: | ||
# component: | ||
# - datacatalog | ||
# - flyteadmin | ||
# - flytecopilot | ||
# - flytepropeller | ||
# - flytescheduler | ||
# uses: ./.github/workflows/publish.yml | ||
# with: | ||
# version: "test-version-monorepo" | ||
# component: ${{ matrix.component }} | ||
# dockerfile: Dockerfile.${{ matrix.component }} | ||
# # TODO(monorepo): set push to true once we depend on bump_version | ||
# # push: true | ||
# push: false | ||
# secrets: | ||
# FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} | ||
# FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
name: Build Docker Image | ||
|
||
on: | ||
workflow_call: | ||
inputs: | ||
component: | ||
required: true | ||
type: string | ||
outputs: | ||
cache_key: | ||
description: "Docker Cache key" | ||
value: ${{ jobs.build_docker.outputs.cache_key }} | ||
jobs: | ||
build_docker: | ||
runs-on: ubuntu-latest | ||
outputs: | ||
cache_key: ${{ steps.cache_key.outputs.cache_key }} | ||
steps: | ||
- name: Checkout | ||
uses: actions/checkout@v3 | ||
- id: load-docker-cache | ||
name: Load Docker Cache | ||
uses: actions/cache@v3 | ||
with: | ||
path: /tmp/tmp/docker-images-${{ inputs.component }} | ||
key: /tmp/docker-images-${{ github.run_id }}-${{ inputs.component }} | ||
restore-keys: | | ||
/tmp/docker-images- | ||
- name: Set cache key output | ||
id: cache_key | ||
run: | | ||
echo ::set-output name=cache_key::/tmp/docker-images-${{ github.run_id }} | ||
- name: Prime docker cache | ||
run: (docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true) && (docker load -i /tmp/tmp/docker-images/snapshot.tar || true) | ||
- name: Build dockerfile | ||
env: | ||
# We are unable to leverage docker buildx here without changing the | ||
# caching mechanism significantly. See here for the caching options | ||
# available for docker buildx: https://docs.docker.com/engine/reference/commandline/buildx_build/#cache-from | ||
# For now at least enable DOCKER_BUILDKIT for faster builds. Eventually we | ||
# should rewrite this pipeline to use docker buildx with cache type=gha. | ||
DOCKER_BUILDKIT: "1" | ||
run: | | ||
docker build -t ${{ github.repository_owner }}/${{ inputs.component }}:builder --target builder --cache-from=${{ github.repository_owner }}/${{ inputs.component }}:builder --file Dockerfile.${{ inputs.component }} . | ||
docker build -t ${{ github.repository_owner }}/${{ inputs.component }}:latest --cache-from=${{ github.repository_owner }}/${{ inputs.component }}:builder --file Dockerfile.${{ inputs.component }} . | ||
- name: Tag and cache docker image | ||
run: | | ||
mkdir -p /tmp/tmp/docker-images-${{ inputs.component }} && docker save ${{ github.repository_owner }}/${{ inputs.component }}:builder -o /tmp/tmp/docker-images-${{ inputs.component }}/snapshot-builder-${{ inputs.component }}.tar && docker save ${{ github.repository_owner }}/${{ inputs.component }}:latest -o /tmp/tmp/docker-images-${{ inputs.component }}/snapshot-${{ inputs.component }}.tar |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,134 @@ | ||
name: End to End tests | ||
|
||
on: | ||
workflow_call: | ||
inputs: | ||
priorities: | ||
description: "Priorities of tests to register (comma-separated)" | ||
required: true | ||
type: string | ||
cache_key: | ||
description: "Cache key for docker image" | ||
required: true | ||
type: string | ||
jobs: | ||
endtoend: | ||
name: End to End tests | ||
runs-on: ubuntu-latest | ||
env: | ||
FLYTESNACKS_VERSION: "" | ||
steps: | ||
- name: Set latest Flytesnacks release | ||
if: ${{ env.FLYTESNACKS_VERSION == '' }} | ||
run: | | ||
FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)" | ||
echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV} | ||
- name: Checkout | ||
uses: actions/checkout@v3 | ||
- uses: unionai/[email protected] | ||
name: Setup flytectl | ||
- uses: actions/setup-python@v3 | ||
with: | ||
python-version: 3.11 | ||
- id: load-docker-cache-datacatalog | ||
uses: actions/cache@v3 | ||
with: | ||
path: /tmp/tmp/docker-images-datacatalog | ||
key: ${{ inputs.cache_key }}-datacatalog | ||
- id: load-docker-cache-flyteadmin | ||
uses: actions/cache@v3 | ||
with: | ||
path: /tmp/tmp/docker-images-flyteadmin | ||
key: ${{ inputs.cache_key }}-flyteadmin | ||
- id: load-docker-cache-flytecopilot | ||
uses: actions/cache@v3 | ||
with: | ||
path: /tmp/tmp/docker-images-flytecopilot | ||
key: ${{ inputs.cache_key }}-flytecopilot | ||
- id: load-docker-cache-flytepropeller | ||
uses: actions/cache@v3 | ||
with: | ||
path: /tmp/tmp/docker-images-flytepropeller | ||
key: ${{ inputs.cache_key }}-flytepropeller | ||
- name: Create Sandbox Cluster | ||
run: | | ||
cp /tmp/tmp/docker-images-datacatalog/snapshot-datacatalog.tar snapshot-datacatalog.tar | ||
cp /tmp/tmp/docker-images-flyteadmin/snapshot-flyteadmin.tar snapshot-flyteadmin.tar | ||
cp /tmp/tmp/docker-images-flytecopilot/snapshot-flytecopilot.tar snapshot-flytecopilot.tar | ||
cp /tmp/tmp/docker-images-flytepropeller/snapshot-flytepropeller.tar snapshot-flytepropeller.tar | ||
flytectl config init | ||
flytectl sandbox start --source=$(pwd) | ||
- name: Prime docker cache | ||
run: | | ||
flytectl sandbox exec -- docker load -i /root/snapshot-datacatalog.tar | ||
flytectl sandbox exec -- docker load -i /root/snapshot-flyteadmin.tar | ||
flytectl sandbox exec -- docker load -i /root/snapshot-flytecopilot.tar | ||
flytectl sandbox exec -- docker load -i /root/snapshot-flytepropeller.tar | ||
- name: Setup Flytekit | ||
run: | | ||
python -m pip install --upgrade pip | ||
pip install flytekit flytekitplugins-deck-standard | ||
pip freeze | ||
- name: Checkout flytesnacks | ||
if: ${{ inputs.priorities == 'P0' }} | ||
uses: actions/checkout@v3 | ||
with: | ||
repository: flyteorg/flytesnacks | ||
path: flytesnacks | ||
ref: ${{ env.FLYTESNACKS_VERSION }} | ||
- name: Register P0 tests | ||
if: ${{ inputs.priorities == 'P0' }} | ||
run: | | ||
for f in \ | ||
basics/basics/hello_world.py \ | ||
basics/basics/workflow.py \ | ||
basics/basics/named_outputs.py \ | ||
advanced_composition/advanced_composition/chain_entities.py \ | ||
advanced_composition/advanced_composition/dynamics.py \ | ||
advanced_composition/advanced_composition/map_task.py \ | ||
advanced_composition/advanced_composition/subworkflows.py \ | ||
data_types_and_io/data_types_and_io/custom_objects.py \ | ||
data_types_and_io/data_types_and_io/schema.py \ | ||
data_types_and_io/data_types_and_io/typed_schema.py ; | ||
do | ||
pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ | ||
register \ | ||
--project flytesnacks \ | ||
--domain development \ | ||
--image cr.flyte.org/flyteorg/flytekit:py3.11-latest \ | ||
--version ${{ env.FLYTESNACKS_VERSION }} \ | ||
flytesnacks/examples/$f; | ||
done | ||
- name: Register all flytesnacks examples | ||
if: ${{ inputs.priorities != 'P0' }} | ||
uses: unionai/[email protected] | ||
with: | ||
flytesnacks: true | ||
project: flytesnacks | ||
version: ${{ env.FLYTESNACKS_VERSION }} | ||
domain: development | ||
# - name: Pre Upgrade Tests | ||
# if: ${{ github.event.repository.name == 'flyteadmin' }} | ||
# env: | ||
# PRIORITIES: "${{ inputs.priorities }}" | ||
# run: | | ||
# make end2end_execute | ||
- name: Upgrade Helm charts | ||
run: | | ||
flytectl sandbox exec -- helm repo add flyteorg https://flyteorg.github.io/flyte | ||
flytectl sandbox exec -- helm repo update | ||
flytectl sandbox exec -- helm upgrade flyte -n flyte-core --kubeconfig=/etc/rancher/k3s/k3s.yaml flyteorg/flyte-core -f /flyteorg/share/flyte/values-sandbox.yaml --wait --set datacatalog.image.repository=${{ github.repository_owner }}/datacalog,datacatalog.image.tag=latest | ||
# TODO(monorepo): the following commands are not correct. | ||
# we have to separate the calls to replace the images into different commands. | ||
# flytectl sandbox exec -- helm upgrade flyte -n flyte-core --kubeconfig=/etc/rancher/k3s/k3s.yaml flyteorg/flyte-core -f /flyteorg/share/flyte/values-sandbox.yaml --wait \ | ||
# --set datacatalog.image.repository=${{ github.repository_owner }}/datacalog,datacatalog.image.tag=latest | ||
# --set flyteadmin.image.repository=${{ github.repository_owner }}/datacalog,flyteadmin.image.tag=latest | ||
# --set flytecopilot.image.repository=${{ github.repository_owner }}/datacalog,flytecopilot.image.tag=latest | ||
# --set flytepropeller.image.repository=${{ github.repository_owner }}/datacalog,flytepropeller.image.tag=latest | ||
flytectl sandbox exec -- k3s kubectl get pods -n flyte -oyaml | ||
- name: Post Upgrade Tests | ||
env: | ||
PRIORITIES: "${{ inputs.priorities }}" | ||
run: | | ||
make end2end_execute |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
name: Go Generate | ||
|
||
on: | ||
workflow_call: | ||
inputs: | ||
component: | ||
required: true | ||
type: string | ||
go-version: | ||
required: true | ||
type: string | ||
jobs: | ||
generate: | ||
runs-on: ubuntu-latest | ||
name: Go Generate | ||
defaults: | ||
run: | ||
working-directory: ${{ inputs.component }} | ||
steps: | ||
- uses: actions/checkout@v3 | ||
- uses: arduino/setup-protoc@v1 | ||
- uses: bufbuild/buf-setup-action@v1 | ||
- uses: actions/setup-go@v3 | ||
with: | ||
go-version: ${{ inputs.go-version }} | ||
- name: Go generate and diff | ||
run: DELTA_CHECK=true make generate |
Oops, something went wrong.