From 3963f4a23acd6fea2f2f8f1a760b44e0f1a90f44 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 07:31:48 +0800 Subject: [PATCH 01/47] test ci Signed-off-by: Kevin Su --- .github/workflows/checks.yml | 488 +++++++++++++-------------- .github/workflows/checks_new.yml | 53 +++ .github/workflows/monodocs_build.yml | 108 +++--- 3 files changed, 351 insertions(+), 298 deletions(-) create mode 100644 .github/workflows/checks_new.yml diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 5a3faabb3..a1abc09e2 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,244 +1,244 @@ -name: Master - -on: - push: - branches: - - master - paths-ignore: - - "docs/**" - pull_request: - paths-ignore: - - "docs/**" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - lint: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.11"] - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Cache pip - uses: actions/cache@v2 - with: - # This path is specific to Ubuntu - path: ~/.cache/pip - # Look to see if there is a cache hit for the corresponding requirements files - key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt')) }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r dev-requirements.txt - - name: Pip info - run: python -m pip list - - name: Lint - run: pre-commit run --all --show-diff-on-failure - - # This is the build system for the new example directory structure - list_examples: - runs-on: ubuntu-latest - name: "Create a list of example packages" - steps: - - uses: actions/checkout@v2 - - name: "Provide the list" - id: create-example-list - run: echo "PACKAGES=$(find examples -mindepth 1 -maxdepth 2 -type f -name Dockerfile -exec dirname '{}' \; | sort | jq --raw-input . | jq --slurp . | jq -c .)" >> "$GITHUB_OUTPUT" - outputs: - packages: "${{ steps.create-example-list.outputs.PACKAGES }}" - - trigger_serialize_register_examples: - name: Serialize & Register Flytesnacks workflow - needs: [list_examples] - uses: ./.github/workflows/serialize_example.yml - with: - packages: ${{ needs.list_examples.outputs.PACKAGES }} - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - push_example_image_to_github: - name: Build & Push Example Image to GHCR - runs-on: ubuntu-latest - needs: [list_examples] - strategy: - matrix: - directory: "${{ fromJson(needs.list_examples.outputs.packages) }}" - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: "Set example name" - id: example_id - run: echo "EXAMPLE_NAME=$(basename -- ${{ matrix.directory }})" >> "$GITHUB_OUTPUT" - - name: Build & Push Docker Image to Github Registry - uses: whoan/docker-build-with-cache-action@v5 - with: - # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - image_name: ${{ github.repository_owner }}/flytecookbook - image_tag: ${{ steps.example_id.outputs.EXAMPLE_NAME }}-latest,${{ steps.example_id.outputs.EXAMPLE_NAME }}-${{ github.sha }} - registry: ghcr.io - push_git_tag: ${{ github.event_name != 'pull_request' }} - push_image_and_stages: ${{ github.event_name != 'pull_request' }} - build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:${{ steps.example_id.outputs.id }}-${{ github.sha }}" - context: ${{ matrix.directory }} - dockerfile: Dockerfile - - bump_version: - name: Bump Version - if: ${{ github.event_name != 'pull_request' }} - needs: [trigger_serialize_register_examples, push_example_image_to_github] # Only to ensure it can successfully build - uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - prerelease: - name: Create Prerelease - needs: [bump_version] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Create Pre release - id: prerelease - run: | - RELEASE_ID=$(curl --location -v -X POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases' \ - --header 'Accept: application/vnd.github.v3+json' \ - --header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \ - --data-raw '{ - "tag_name": "'${{ needs.bump-version.outputs.version }}'", - "prerelease": true - }' | jq -r '.id') - echo ::set-output name=release_id::$RELEASE_ID - outputs: - release_id: ${{ steps.prerelease.outputs.release_id }} - - # Download artifacts again and push them to the release only if this is not a pull request - release_workflow: - name: Publish artifacts to github release - runs-on: ubuntu-latest - needs: [prerelease] - strategy: - matrix: - python-version: ["3.11"] - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: setup download artifact dir - run: | - mkdir download-artifact - - name: Download artifacts - uses: actions/download-artifact@v2 - with: - path: ./download-artifact/ - - name: Package Examples - run: | - mkdir -p release-snacks - cd download-artifact - for i in */; do tar -czvf "../release-snacks/${i%/}.tar.gz" "$i" & done; wait - cd .. && sudo rm -rf download-artifact/ - cp flyte_tests_manifest.json release-snacks/flyte_tests_manifest.json - cp flyte_tests.txt release-snacks/flyte_tests.txt - - name: Release test manifest - uses: goreleaser/goreleaser-action@v2 - with: - version: latest - args: release --rm-dist - env: - GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} - GORELEASER_CURRENT_TAG: ${{ needs.bump-version.outputs.version }} - - make_release: - name: Mark github pre-release as Release - needs: [release_workflow] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Update Release - id: release - run: | - curl --location -X -v POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases/${{ needs.prerelease.outputs.release_id }}' \ - --header 'Accept: application/vnd.github.v3+json' \ - --header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \ - --data-raw '{ - "tag_name": "'${{ needs.bump-version.outputs.version }}'", - "prerelease": false - }' - - e2e-tests: - runs-on: ubuntu-latest - env: - FLYTESNACKS_PRIORITIES: "P0" - FLYTESNACKS_VERSION: "" - timeout-minutes: 30 - steps: - - uses: insightsengineering/disk-space-reclaimer@v1 - - name: Set latest Flytesnacks release - if: ${{ env.FLYTESNACKS_VERSION == '' }} - run: | - FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)" - echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV} - - name: Checkout - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - uses: unionai/flytectl-setup-action@v0.0.1 - - name: Setup sandbox - run: | - mkdir -p ~/.flyte/sandbox - cat << EOF > ~/.flyte/sandbox/config.yaml - task_resources: - defaults: - cpu: "0" - memory: "0" - limits: - cpu: "0" - memory: "0" - EOF - flytectl demo start --imagePullPolicy Never - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install flytekit flytekitplugins-deck-standard torch - pip freeze - - name: Checkout flytesnacks - uses: actions/checkout@v3 - with: - repository: flyteorg/flytesnacks - path: flytesnacks - - name: Verify existence of the tests - run: | - python flyte_tests_validate.py - - name: Register specific tests - run: | - while read -r line; - do - pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ - register \ - --project flytesnacks \ - --domain development \ - --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \ - --version ${{ env.FLYTESNACKS_VERSION }} \ - flytesnacks/$line; - done < flyte_tests.txt - - name: Setup tmate session - if: ${{ failure() }} - uses: mxschmitt/action-tmate@v3 +#name: Master +# +#on: +# push: +# branches: +# - master +# paths-ignore: +# - "docs/**" +# pull_request: +# paths-ignore: +# - "docs/**" +# +#concurrency: +# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} +# cancel-in-progress: true +# +#jobs: +# lint: +# runs-on: ubuntu-latest +# strategy: +# matrix: +# python-version: ["3.11"] +# steps: +# - uses: actions/checkout@v2 +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v2 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Cache pip +# uses: actions/cache@v2 +# with: +# # This path is specific to Ubuntu +# path: ~/.cache/pip +# # Look to see if there is a cache hit for the corresponding requirements files +# key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt')) }} +# - name: Install dependencies +# run: | +# python -m pip install --upgrade pip +# pip install -r dev-requirements.txt +# - name: Pip info +# run: python -m pip list +# - name: Lint +# run: pre-commit run --all --show-diff-on-failure +# +# # This is the build system for the new example directory structure +# list_examples: +# runs-on: ubuntu-latest +# name: "Create a list of example packages" +# steps: +# - uses: actions/checkout@v2 +# - name: "Provide the list" +# id: create-example-list +# run: echo "PACKAGES=$(find examples -mindepth 1 -maxdepth 2 -type f -name Dockerfile -exec dirname '{}' \; | sort | jq --raw-input . | jq --slurp . | jq -c .)" >> "$GITHUB_OUTPUT" +# outputs: +# packages: "${{ steps.create-example-list.outputs.PACKAGES }}" +# +# trigger_serialize_register_examples: +# name: Serialize & Register Flytesnacks workflow +# needs: [list_examples] +# uses: ./.github/workflows/serialize_example.yml +# with: +# packages: ${{ needs.list_examples.outputs.PACKAGES }} +# secrets: +# FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} +# +# push_example_image_to_github: +# name: Build & Push Example Image to GHCR +# runs-on: ubuntu-latest +# needs: [list_examples] +# strategy: +# matrix: +# directory: "${{ fromJson(needs.list_examples.outputs.packages) }}" +# steps: +# - uses: actions/checkout@v2 +# with: +# fetch-depth: "0" +# - name: "Set example name" +# id: example_id +# run: echo "EXAMPLE_NAME=$(basename -- ${{ matrix.directory }})" >> "$GITHUB_OUTPUT" +# - name: Build & Push Docker Image to Github Registry +# uses: whoan/docker-build-with-cache-action@v5 +# with: +# # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package +# username: "${{ secrets.FLYTE_BOT_USERNAME }}" +# password: "${{ secrets.FLYTE_BOT_PAT }}" +# image_name: ${{ github.repository_owner }}/flytecookbook +# image_tag: ${{ steps.example_id.outputs.EXAMPLE_NAME }}-latest,${{ steps.example_id.outputs.EXAMPLE_NAME }}-${{ github.sha }} +# registry: ghcr.io +# push_git_tag: ${{ github.event_name != 'pull_request' }} +# push_image_and_stages: ${{ github.event_name != 'pull_request' }} +# build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:${{ steps.example_id.outputs.id }}-${{ github.sha }}" +# context: ${{ matrix.directory }} +# dockerfile: Dockerfile +# +# bump_version: +# name: Bump Version +# if: ${{ github.event_name != 'pull_request' }} +# needs: [trigger_serialize_register_examples, push_example_image_to_github] # Only to ensure it can successfully build +# uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master +# secrets: +# FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} +# +# prerelease: +# name: Create Prerelease +# needs: [bump_version] +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v2 +# with: +# fetch-depth: "0" +# - name: Create Pre release +# id: prerelease +# run: | +# RELEASE_ID=$(curl --location -v -X POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases' \ +# --header 'Accept: application/vnd.github.v3+json' \ +# --header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \ +# --data-raw '{ +# "tag_name": "'${{ needs.bump-version.outputs.version }}'", +# "prerelease": true +# }' | jq -r '.id') +# echo ::set-output name=release_id::$RELEASE_ID +# outputs: +# release_id: ${{ steps.prerelease.outputs.release_id }} +# +# # Download artifacts again and push them to the release only if this is not a pull request +# release_workflow: +# name: Publish artifacts to github release +# runs-on: ubuntu-latest +# needs: [prerelease] +# strategy: +# matrix: +# python-version: ["3.11"] +# steps: +# - uses: actions/checkout@v2 +# with: +# fetch-depth: "0" +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v2 +# with: +# python-version: ${{ matrix.python-version }} +# - name: setup download artifact dir +# run: | +# mkdir download-artifact +# - name: Download artifacts +# uses: actions/download-artifact@v2 +# with: +# path: ./download-artifact/ +# - name: Package Examples +# run: | +# mkdir -p release-snacks +# cd download-artifact +# for i in */; do tar -czvf "../release-snacks/${i%/}.tar.gz" "$i" & done; wait +# cd .. && sudo rm -rf download-artifact/ +# cp flyte_tests_manifest.json release-snacks/flyte_tests_manifest.json +# cp flyte_tests.txt release-snacks/flyte_tests.txt +# - name: Release test manifest +# uses: goreleaser/goreleaser-action@v2 +# with: +# version: latest +# args: release --rm-dist +# env: +# GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} +# GORELEASER_CURRENT_TAG: ${{ needs.bump-version.outputs.version }} +# +# make_release: +# name: Mark github pre-release as Release +# needs: [release_workflow] +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v2 +# with: +# fetch-depth: "0" +# - name: Update Release +# id: release +# run: | +# curl --location -X -v POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases/${{ needs.prerelease.outputs.release_id }}' \ +# --header 'Accept: application/vnd.github.v3+json' \ +# --header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \ +# --data-raw '{ +# "tag_name": "'${{ needs.bump-version.outputs.version }}'", +# "prerelease": false +# }' +# +# e2e-tests: +# runs-on: ubuntu-latest +# env: +# FLYTESNACKS_PRIORITIES: "P0" +# FLYTESNACKS_VERSION: "" +# timeout-minutes: 30 +# steps: +# - uses: insightsengineering/disk-space-reclaimer@v1 +# - name: Set latest Flytesnacks release +# if: ${{ env.FLYTESNACKS_VERSION == '' }} +# run: | +# FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)" +# echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV} +# - name: Checkout +# uses: actions/checkout@v3 +# - name: Set up Python +# uses: actions/setup-python@v4 +# with: +# python-version: "3.11" +# - uses: unionai/flytectl-setup-action@v0.0.1 +# - name: Setup sandbox +# run: | +# mkdir -p ~/.flyte/sandbox +# cat << EOF > ~/.flyte/sandbox/config.yaml +# task_resources: +# defaults: +# cpu: "0" +# memory: "0" +# limits: +# cpu: "0" +# memory: "0" +# EOF +# flytectl demo start --imagePullPolicy Never +# - name: Install Python dependencies +# run: | +# python -m pip install --upgrade pip +# pip install flytekit flytekitplugins-deck-standard torch +# pip freeze +# - name: Checkout flytesnacks +# uses: actions/checkout@v3 +# with: +# repository: flyteorg/flytesnacks +# path: flytesnacks +# - name: Verify existence of the tests +# run: | +# python flyte_tests_validate.py +# - name: Register specific tests +# run: | +# while read -r line; +# do +# pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ +# register \ +# --project flytesnacks \ +# --domain development \ +# --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \ +# --version ${{ env.FLYTESNACKS_VERSION }} \ +# flytesnacks/$line; +# done < flyte_tests.txt +# - name: Setup tmate session +# if: ${{ failure() }} +# uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml new file mode 100644 index 000000000..e1b297a3e --- /dev/null +++ b/.github/workflows/checks_new.yml @@ -0,0 +1,53 @@ +name: Master + +on: + push: + branches: + - master + paths-ignore: + - "docs/**" + pull_request: + paths-ignore: + - "docs/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + serialize_examples: + runs-on: "ubuntu-latest" + name: Serialize Flytesnacks workflow + strategy: + matrix: + example: ["examples/k8s_spark_plugin"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Cache pip + uses: actions/cache@v3 + with: + # This path is specific to Ubuntu + path: ~/.cache/pip + # Look to see if there is a cache hit for the corresponding requirements files + key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }} + - name: Install dependencies + run: | + pip install uv + uv venv + source .venv/bin/activate + pip install flytekitplugins-spark flytekitplugins-envd + - name: Login to GitHub Container Registry + if: ${{ github.event_name == 'push' }} + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ secrets.FLYTE_BOT_USERNAME }} + password: ${{ secrets.FLYTE_BOT_PAT }} + - name: Pyflyte package + run: | + cd ${{ matrix.example }} + pyflyte --pkgs k8s_spark_plugin package -o spark-package.tgz --force --fast diff --git a/.github/workflows/monodocs_build.yml b/.github/workflows/monodocs_build.yml index 833910e76..eb5c8021b 100644 --- a/.github/workflows/monodocs_build.yml +++ b/.github/workflows/monodocs_build.yml @@ -1,54 +1,54 @@ -name: Monodocs Build - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -on: - push: - branches: - - master - pull_request: - branches: - - master -jobs: - docs: - name: Docs Build - runs-on: ubuntu-latest - steps: - - name: Fetch flytesnacks code - uses: actions/checkout@v4 - with: - path: "${{ github.workspace }}/flytesnacks" - - name: Fetch flyte code - uses: actions/checkout@v4 - with: - repository: flyteorg/flyte - path: "${{ github.workspace }}/flyte" - - uses: conda-incubator/setup-miniconda@v3 - with: - auto-update-conda: true - python-version: 3.9 - - shell: bash -el {0} - working-directory: ${{ github.workspace }}/flyte - run: | - conda install -c conda-forge conda-lock - conda-lock install -n monodocs-env monodocs-environment.lock.yaml - - shell: bash -el {0} - working-directory: ${{ github.workspace }}/flyte - run: | - conda activate monodocs-env - pip install ./flyteidl - conda info - conda list - conda config --show-sources - conda config --show - printenv | sort - - name: Build the documentation - working-directory: ${{ github.workspace }}/flyte - shell: bash -el {0} - env: - FLYTESNACKS_LOCAL_PATH: ${{ github.workspace }}/flytesnacks - run: | - conda activate monodocs-env - make -C docs html SPHINXOPTS="-W -vvv" +#name: Monodocs Build +# +#concurrency: +# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} +# cancel-in-progress: true +# +#on: +# push: +# branches: +# - master +# pull_request: +# branches: +# - master +#jobs: +# docs: +# name: Docs Build +# runs-on: ubuntu-latest +# steps: +# - name: Fetch flytesnacks code +# uses: actions/checkout@v4 +# with: +# path: "${{ github.workspace }}/flytesnacks" +# - name: Fetch flyte code +# uses: actions/checkout@v4 +# with: +# repository: flyteorg/flyte +# path: "${{ github.workspace }}/flyte" +# - uses: conda-incubator/setup-miniconda@v3 +# with: +# auto-update-conda: true +# python-version: 3.9 +# - shell: bash -el {0} +# working-directory: ${{ github.workspace }}/flyte +# run: | +# conda install -c conda-forge conda-lock +# conda-lock install -n monodocs-env monodocs-environment.lock.yaml +# - shell: bash -el {0} +# working-directory: ${{ github.workspace }}/flyte +# run: | +# conda activate monodocs-env +# pip install ./flyteidl +# conda info +# conda list +# conda config --show-sources +# conda config --show +# printenv | sort +# - name: Build the documentation +# working-directory: ${{ github.workspace }}/flyte +# shell: bash -el {0} +# env: +# FLYTESNACKS_LOCAL_PATH: ${{ github.workspace }}/flytesnacks +# run: | +# conda activate monodocs-env +# make -C docs html SPHINXOPTS="-W -vvv" From af61e2c324ed451f0bc1f06560a0b3c5ef286044 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 07:32:02 +0800 Subject: [PATCH 02/47] test ci Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index e1b297a3e..98c1a30ba 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -4,6 +4,7 @@ on: push: branches: - master + - ci-overhaul paths-ignore: - "docs/**" pull_request: From 32452bc00df90d704a85f1d5c26a3aaada2d130b Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 07:34:38 +0800 Subject: [PATCH 03/47] test pre Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 98c1a30ba..5de9dfa9f 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -40,7 +40,7 @@ jobs: pip install uv uv venv source .venv/bin/activate - pip install flytekitplugins-spark flytekitplugins-envd + pip install --pre flytekitplugins-spark flytekitplugins-envd - name: Login to GitHub Container Registry if: ${{ github.event_name == 'push' }} uses: docker/login-action@v1 From 481e6fe28cd8501f55e14075497f1fb549abc136 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 07:37:47 +0800 Subject: [PATCH 04/47] test Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 5de9dfa9f..33f73f1a9 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -18,10 +18,10 @@ concurrency: jobs: serialize_examples: runs-on: "ubuntu-latest" - name: Serialize Flytesnacks workflow + name: Serialize strategy: matrix: - example: ["examples/k8s_spark_plugin"] + example: ["k8s_spark_plugin"] steps: - uses: actions/checkout@v4 - name: Set up Python @@ -40,7 +40,7 @@ jobs: pip install uv uv venv source .venv/bin/activate - pip install --pre flytekitplugins-spark flytekitplugins-envd + pip install --pre numpy<2.0.0 flytekitplugins-spark flytekitplugins-envd - name: Login to GitHub Container Registry if: ${{ github.event_name == 'push' }} uses: docker/login-action@v1 @@ -50,5 +50,5 @@ jobs: password: ${{ secrets.FLYTE_BOT_PAT }} - name: Pyflyte package run: | - cd ${{ matrix.example }} + cd examples/${{ matrix.example }} pyflyte --pkgs k8s_spark_plugin package -o spark-package.tgz --force --fast From 8a16857b20d7bd2d315580a0746be1d25252e992 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 07:38:44 +0800 Subject: [PATCH 05/47] test Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 33f73f1a9..b33f37857 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -40,7 +40,7 @@ jobs: pip install uv uv venv source .venv/bin/activate - pip install --pre numpy<2.0.0 flytekitplugins-spark flytekitplugins-envd + pip install --pre "numpy<2.0.0" flytekitplugins-spark flytekitplugins-envd - name: Login to GitHub Container Registry if: ${{ github.event_name == 'push' }} uses: docker/login-action@v1 From 7ce9331988a20bf0be35df15680e35d7f1305f6f Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 07:46:00 +0800 Subject: [PATCH 06/47] test ci Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index b33f37857..a4080082b 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -42,7 +42,6 @@ jobs: source .venv/bin/activate pip install --pre "numpy<2.0.0" flytekitplugins-spark flytekitplugins-envd - name: Login to GitHub Container Registry - if: ${{ github.event_name == 'push' }} uses: docker/login-action@v1 with: registry: ghcr.io From 1d393450b4bd0b18bd81eea32b42dd8a2bc704dd Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 07:55:17 +0800 Subject: [PATCH 07/47] test airflow Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index a4080082b..5b909824f 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -4,7 +4,6 @@ on: push: branches: - master - - ci-overhaul paths-ignore: - "docs/**" pull_request: @@ -21,7 +20,10 @@ jobs: name: Serialize strategy: matrix: - example: ["k8s_spark_plugin"] + example: + - advanced_composition + - airflow_agent + - k8s_spark_plugin steps: - uses: actions/checkout@v4 - name: Set up Python @@ -40,7 +42,7 @@ jobs: pip install uv uv venv source .venv/bin/activate - pip install --pre "numpy<2.0.0" flytekitplugins-spark flytekitplugins-envd + pip install "numpy<2.0.0" flytekitplugins-spark flytekitplugins-envd flytekitplugins-airflow - name: Login to GitHub Container Registry uses: docker/login-action@v1 with: From 0faa0141dbde40a2be22347ea77fb9e7069464ea Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 08:02:47 +0800 Subject: [PATCH 08/47] test Signed-off-by: Kevin Su --- .github/workflows/checks.yml | 482 +++++++++++++-------------- .github/workflows/checks_new.yml | 2 +- .github/workflows/monodocs_build.yml | 102 +++--- 3 files changed, 293 insertions(+), 293 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index a1abc09e2..13b03ed41 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,244 +1,244 @@ -#name: Master -# -#on: -# push: -# branches: -# - master -# paths-ignore: -# - "docs/**" +name: Master + +on: + push: + branches: + - master + paths-ignore: + - "docs/**" # pull_request: # paths-ignore: # - "docs/**" -# -#concurrency: -# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} -# cancel-in-progress: true -# -#jobs: -# lint: -# runs-on: ubuntu-latest -# strategy: -# matrix: -# python-version: ["3.11"] -# steps: -# - uses: actions/checkout@v2 -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v2 -# with: -# python-version: ${{ matrix.python-version }} -# - name: Cache pip -# uses: actions/cache@v2 -# with: -# # This path is specific to Ubuntu -# path: ~/.cache/pip -# # Look to see if there is a cache hit for the corresponding requirements files -# key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt')) }} -# - name: Install dependencies -# run: | -# python -m pip install --upgrade pip -# pip install -r dev-requirements.txt -# - name: Pip info -# run: python -m pip list -# - name: Lint -# run: pre-commit run --all --show-diff-on-failure -# -# # This is the build system for the new example directory structure -# list_examples: -# runs-on: ubuntu-latest -# name: "Create a list of example packages" -# steps: -# - uses: actions/checkout@v2 -# - name: "Provide the list" -# id: create-example-list -# run: echo "PACKAGES=$(find examples -mindepth 1 -maxdepth 2 -type f -name Dockerfile -exec dirname '{}' \; | sort | jq --raw-input . | jq --slurp . | jq -c .)" >> "$GITHUB_OUTPUT" -# outputs: -# packages: "${{ steps.create-example-list.outputs.PACKAGES }}" -# -# trigger_serialize_register_examples: -# name: Serialize & Register Flytesnacks workflow -# needs: [list_examples] -# uses: ./.github/workflows/serialize_example.yml -# with: -# packages: ${{ needs.list_examples.outputs.PACKAGES }} -# secrets: -# FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} -# -# push_example_image_to_github: -# name: Build & Push Example Image to GHCR -# runs-on: ubuntu-latest -# needs: [list_examples] -# strategy: -# matrix: -# directory: "${{ fromJson(needs.list_examples.outputs.packages) }}" -# steps: -# - uses: actions/checkout@v2 -# with: -# fetch-depth: "0" -# - name: "Set example name" -# id: example_id -# run: echo "EXAMPLE_NAME=$(basename -- ${{ matrix.directory }})" >> "$GITHUB_OUTPUT" -# - name: Build & Push Docker Image to Github Registry -# uses: whoan/docker-build-with-cache-action@v5 -# with: -# # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package -# username: "${{ secrets.FLYTE_BOT_USERNAME }}" -# password: "${{ secrets.FLYTE_BOT_PAT }}" -# image_name: ${{ github.repository_owner }}/flytecookbook -# image_tag: ${{ steps.example_id.outputs.EXAMPLE_NAME }}-latest,${{ steps.example_id.outputs.EXAMPLE_NAME }}-${{ github.sha }} -# registry: ghcr.io -# push_git_tag: ${{ github.event_name != 'pull_request' }} -# push_image_and_stages: ${{ github.event_name != 'pull_request' }} -# build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:${{ steps.example_id.outputs.id }}-${{ github.sha }}" -# context: ${{ matrix.directory }} -# dockerfile: Dockerfile -# -# bump_version: -# name: Bump Version -# if: ${{ github.event_name != 'pull_request' }} -# needs: [trigger_serialize_register_examples, push_example_image_to_github] # Only to ensure it can successfully build -# uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master -# secrets: -# FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} -# -# prerelease: -# name: Create Prerelease -# needs: [bump_version] -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v2 -# with: -# fetch-depth: "0" -# - name: Create Pre release -# id: prerelease -# run: | -# RELEASE_ID=$(curl --location -v -X POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases' \ -# --header 'Accept: application/vnd.github.v3+json' \ -# --header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \ -# --data-raw '{ -# "tag_name": "'${{ needs.bump-version.outputs.version }}'", -# "prerelease": true -# }' | jq -r '.id') -# echo ::set-output name=release_id::$RELEASE_ID -# outputs: -# release_id: ${{ steps.prerelease.outputs.release_id }} -# -# # Download artifacts again and push them to the release only if this is not a pull request -# release_workflow: -# name: Publish artifacts to github release -# runs-on: ubuntu-latest -# needs: [prerelease] -# strategy: -# matrix: -# python-version: ["3.11"] -# steps: -# - uses: actions/checkout@v2 -# with: -# fetch-depth: "0" -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v2 -# with: -# python-version: ${{ matrix.python-version }} -# - name: setup download artifact dir -# run: | -# mkdir download-artifact -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# path: ./download-artifact/ -# - name: Package Examples -# run: | -# mkdir -p release-snacks -# cd download-artifact -# for i in */; do tar -czvf "../release-snacks/${i%/}.tar.gz" "$i" & done; wait -# cd .. && sudo rm -rf download-artifact/ -# cp flyte_tests_manifest.json release-snacks/flyte_tests_manifest.json -# cp flyte_tests.txt release-snacks/flyte_tests.txt -# - name: Release test manifest -# uses: goreleaser/goreleaser-action@v2 -# with: -# version: latest -# args: release --rm-dist -# env: -# GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} -# GORELEASER_CURRENT_TAG: ${{ needs.bump-version.outputs.version }} -# -# make_release: -# name: Mark github pre-release as Release -# needs: [release_workflow] -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v2 -# with: -# fetch-depth: "0" -# - name: Update Release -# id: release -# run: | -# curl --location -X -v POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases/${{ needs.prerelease.outputs.release_id }}' \ -# --header 'Accept: application/vnd.github.v3+json' \ -# --header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \ -# --data-raw '{ -# "tag_name": "'${{ needs.bump-version.outputs.version }}'", -# "prerelease": false -# }' -# -# e2e-tests: -# runs-on: ubuntu-latest -# env: -# FLYTESNACKS_PRIORITIES: "P0" -# FLYTESNACKS_VERSION: "" -# timeout-minutes: 30 -# steps: -# - uses: insightsengineering/disk-space-reclaimer@v1 -# - name: Set latest Flytesnacks release -# if: ${{ env.FLYTESNACKS_VERSION == '' }} -# run: | -# FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)" -# echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV} -# - name: Checkout -# uses: actions/checkout@v3 -# - name: Set up Python -# uses: actions/setup-python@v4 -# with: -# python-version: "3.11" -# - uses: unionai/flytectl-setup-action@v0.0.1 -# - name: Setup sandbox -# run: | -# mkdir -p ~/.flyte/sandbox -# cat << EOF > ~/.flyte/sandbox/config.yaml -# task_resources: -# defaults: -# cpu: "0" -# memory: "0" -# limits: -# cpu: "0" -# memory: "0" -# EOF -# flytectl demo start --imagePullPolicy Never -# - name: Install Python dependencies -# run: | -# python -m pip install --upgrade pip -# pip install flytekit flytekitplugins-deck-standard torch -# pip freeze -# - name: Checkout flytesnacks -# uses: actions/checkout@v3 -# with: -# repository: flyteorg/flytesnacks -# path: flytesnacks -# - name: Verify existence of the tests -# run: | -# python flyte_tests_validate.py -# - name: Register specific tests -# run: | -# while read -r line; -# do -# pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ -# register \ -# --project flytesnacks \ -# --domain development \ -# --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \ -# --version ${{ env.FLYTESNACKS_VERSION }} \ -# flytesnacks/$line; -# done < flyte_tests.txt -# - name: Setup tmate session -# if: ${{ failure() }} -# uses: mxschmitt/action-tmate@v3 + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11"] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Cache pip + uses: actions/cache@v2 + with: + # This path is specific to Ubuntu + path: ~/.cache/pip + # Look to see if there is a cache hit for the corresponding requirements files + key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt')) }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r dev-requirements.txt + - name: Pip info + run: python -m pip list + - name: Lint + run: pre-commit run --all --show-diff-on-failure + + # This is the build system for the new example directory structure + list_examples: + runs-on: ubuntu-latest + name: "Create a list of example packages" + steps: + - uses: actions/checkout@v2 + - name: "Provide the list" + id: create-example-list + run: echo "PACKAGES=$(find examples -mindepth 1 -maxdepth 2 -type f -name Dockerfile -exec dirname '{}' \; | sort | jq --raw-input . | jq --slurp . | jq -c .)" >> "$GITHUB_OUTPUT" + outputs: + packages: "${{ steps.create-example-list.outputs.PACKAGES }}" + + trigger_serialize_register_examples: + name: Serialize & Register Flytesnacks workflow + needs: [list_examples] + uses: ./.github/workflows/serialize_example.yml + with: + packages: ${{ needs.list_examples.outputs.PACKAGES }} + secrets: + FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} + + push_example_image_to_github: + name: Build & Push Example Image to GHCR + runs-on: ubuntu-latest + needs: [list_examples] + strategy: + matrix: + directory: "${{ fromJson(needs.list_examples.outputs.packages) }}" + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: "0" + - name: "Set example name" + id: example_id + run: echo "EXAMPLE_NAME=$(basename -- ${{ matrix.directory }})" >> "$GITHUB_OUTPUT" + - name: Build & Push Docker Image to Github Registry + uses: whoan/docker-build-with-cache-action@v5 + with: + # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package + username: "${{ secrets.FLYTE_BOT_USERNAME }}" + password: "${{ secrets.FLYTE_BOT_PAT }}" + image_name: ${{ github.repository_owner }}/flytecookbook + image_tag: ${{ steps.example_id.outputs.EXAMPLE_NAME }}-latest,${{ steps.example_id.outputs.EXAMPLE_NAME }}-${{ github.sha }} + registry: ghcr.io + push_git_tag: ${{ github.event_name != 'pull_request' }} + push_image_and_stages: ${{ github.event_name != 'pull_request' }} + build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:${{ steps.example_id.outputs.id }}-${{ github.sha }}" + context: ${{ matrix.directory }} + dockerfile: Dockerfile + + bump_version: + name: Bump Version + if: ${{ github.event_name != 'pull_request' }} + needs: [trigger_serialize_register_examples, push_example_image_to_github] # Only to ensure it can successfully build + uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master + secrets: + FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} + + prerelease: + name: Create Prerelease + needs: [bump_version] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: "0" + - name: Create Pre release + id: prerelease + run: | + RELEASE_ID=$(curl --location -v -X POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases' \ + --header 'Accept: application/vnd.github.v3+json' \ + --header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \ + --data-raw '{ + "tag_name": "'${{ needs.bump-version.outputs.version }}'", + "prerelease": true + }' | jq -r '.id') + echo ::set-output name=release_id::$RELEASE_ID + outputs: + release_id: ${{ steps.prerelease.outputs.release_id }} + + # Download artifacts again and push them to the release only if this is not a pull request + release_workflow: + name: Publish artifacts to github release + runs-on: ubuntu-latest + needs: [prerelease] + strategy: + matrix: + python-version: ["3.11"] + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: "0" + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: setup download artifact dir + run: | + mkdir download-artifact + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + path: ./download-artifact/ + - name: Package Examples + run: | + mkdir -p release-snacks + cd download-artifact + for i in */; do tar -czvf "../release-snacks/${i%/}.tar.gz" "$i" & done; wait + cd .. && sudo rm -rf download-artifact/ + cp flyte_tests_manifest.json release-snacks/flyte_tests_manifest.json + cp flyte_tests.txt release-snacks/flyte_tests.txt + - name: Release test manifest + uses: goreleaser/goreleaser-action@v2 + with: + version: latest + args: release --rm-dist + env: + GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} + GORELEASER_CURRENT_TAG: ${{ needs.bump-version.outputs.version }} + + make_release: + name: Mark github pre-release as Release + needs: [release_workflow] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: "0" + - name: Update Release + id: release + run: | + curl --location -X -v POST 'https://api.github.com/repos/flyteorg/flytesnacks/releases/${{ needs.prerelease.outputs.release_id }}' \ + --header 'Accept: application/vnd.github.v3+json' \ + --header 'Authorization: Bearer ${{ secrets.FLYTE_BOT_PAT }}' \ + --data-raw '{ + "tag_name": "'${{ needs.bump-version.outputs.version }}'", + "prerelease": false + }' + + e2e-tests: + runs-on: ubuntu-latest + env: + FLYTESNACKS_PRIORITIES: "P0" + FLYTESNACKS_VERSION: "" + timeout-minutes: 30 + steps: + - uses: insightsengineering/disk-space-reclaimer@v1 + - name: Set latest Flytesnacks release + if: ${{ env.FLYTESNACKS_VERSION == '' }} + run: | + FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)" + echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV} + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - uses: unionai/flytectl-setup-action@v0.0.1 + - name: Setup sandbox + run: | + mkdir -p ~/.flyte/sandbox + cat << EOF > ~/.flyte/sandbox/config.yaml + task_resources: + defaults: + cpu: "0" + memory: "0" + limits: + cpu: "0" + memory: "0" + EOF + flytectl demo start --imagePullPolicy Never + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install flytekit flytekitplugins-deck-standard torch + pip freeze + - name: Checkout flytesnacks + uses: actions/checkout@v3 + with: + repository: flyteorg/flytesnacks + path: flytesnacks + - name: Verify existence of the tests + run: | + python flyte_tests_validate.py + - name: Register specific tests + run: | + while read -r line; + do + pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ + register \ + --project flytesnacks \ + --domain development \ + --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \ + --version ${{ env.FLYTESNACKS_VERSION }} \ + flytesnacks/$line; + done < flyte_tests.txt + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 5b909824f..1e237eb3c 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -52,4 +52,4 @@ jobs: - name: Pyflyte package run: | cd examples/${{ matrix.example }} - pyflyte --pkgs k8s_spark_plugin package -o spark-package.tgz --force --fast + pyflyte --pkgs ${{ matrix.example }} package -o spark-package.tgz --force --fast diff --git a/.github/workflows/monodocs_build.yml b/.github/workflows/monodocs_build.yml index eb5c8021b..d3bdfeecb 100644 --- a/.github/workflows/monodocs_build.yml +++ b/.github/workflows/monodocs_build.yml @@ -1,54 +1,54 @@ -#name: Monodocs Build -# -#concurrency: -# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} -# cancel-in-progress: true -# -#on: -# push: -# branches: -# - master +name: Monodocs Build + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +on: + push: + branches: + - master # pull_request: # branches: # - master -#jobs: -# docs: -# name: Docs Build -# runs-on: ubuntu-latest -# steps: -# - name: Fetch flytesnacks code -# uses: actions/checkout@v4 -# with: -# path: "${{ github.workspace }}/flytesnacks" -# - name: Fetch flyte code -# uses: actions/checkout@v4 -# with: -# repository: flyteorg/flyte -# path: "${{ github.workspace }}/flyte" -# - uses: conda-incubator/setup-miniconda@v3 -# with: -# auto-update-conda: true -# python-version: 3.9 -# - shell: bash -el {0} -# working-directory: ${{ github.workspace }}/flyte -# run: | -# conda install -c conda-forge conda-lock -# conda-lock install -n monodocs-env monodocs-environment.lock.yaml -# - shell: bash -el {0} -# working-directory: ${{ github.workspace }}/flyte -# run: | -# conda activate monodocs-env -# pip install ./flyteidl -# conda info -# conda list -# conda config --show-sources -# conda config --show -# printenv | sort -# - name: Build the documentation -# working-directory: ${{ github.workspace }}/flyte -# shell: bash -el {0} -# env: -# FLYTESNACKS_LOCAL_PATH: ${{ github.workspace }}/flytesnacks -# run: | -# conda activate monodocs-env -# make -C docs html SPHINXOPTS="-W -vvv" +jobs: + docs: + name: Docs Build + runs-on: ubuntu-latest + steps: + - name: Fetch flytesnacks code + uses: actions/checkout@v4 + with: + path: "${{ github.workspace }}/flytesnacks" + - name: Fetch flyte code + uses: actions/checkout@v4 + with: + repository: flyteorg/flyte + path: "${{ github.workspace }}/flyte" + - uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + python-version: 3.9 + - shell: bash -el {0} + working-directory: ${{ github.workspace }}/flyte + run: | + conda install -c conda-forge conda-lock + conda-lock install -n monodocs-env monodocs-environment.lock.yaml + - shell: bash -el {0} + working-directory: ${{ github.workspace }}/flyte + run: | + conda activate monodocs-env + pip install ./flyteidl + conda info + conda list + conda config --show-sources + conda config --show + printenv | sort + - name: Build the documentation + working-directory: ${{ github.workspace }}/flyte + shell: bash -el {0} + env: + FLYTESNACKS_LOCAL_PATH: ${{ github.workspace }}/flytesnacks + run: | + conda activate monodocs-env + make -C docs html SPHINXOPTS="-W -vvv" From 003747dccfc9a9e0d5d853abf542f4d2b16859ff Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 08:18:12 +0800 Subject: [PATCH 09/47] Add requirements.in Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 4 +++- requirements.in | 36 ++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 requirements.in diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 1e237eb3c..4896900fb 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -24,6 +24,8 @@ jobs: - advanced_composition - airflow_agent - k8s_spark_plugin + - athena_plugin + - aws_batch_plugin steps: - uses: actions/checkout@v4 - name: Set up Python @@ -42,7 +44,7 @@ jobs: pip install uv uv venv source .venv/bin/activate - pip install "numpy<2.0.0" flytekitplugins-spark flytekitplugins-envd flytekitplugins-airflow + uv pip install -r requirements.in - name: Login to GitHub Container Registry uses: docker/login-action@v1 with: diff --git a/requirements.in b/requirements.in new file mode 100644 index 000000000..7e9e45195 --- /dev/null +++ b/requirements.in @@ -0,0 +1,36 @@ +numpy<2.0.0 +flytekitplugins-airflow +flytekitplugins-athena +flytekitplugins-awsbatch +flytekitplugins-awssagemaker +flytekitplugins-bigquery +flytekitplugins-dask +flytekitplugins-dbt +flytekitplugins-deck-standard +flytekitplugins.dolt +flytekitplugins-duckdb +flytekitplugins-envd +flytekitplugins-great-expectations +flytekitplugins-hive +flytekitplugins-huggingface +flytekitplugins-pod +flytekitplugins-kfmpi +flytekitplugins-kfpytorch +flytekitplugins-kftensorflow +flytekitplugins-mlflow +flytekitplugins-mmcloud +flytekitplugins-modin +flytekitplugins-onnxpytorch +flytekitplugins-onnxscikitlearn +flytekitplugins-onnxtensorflow +flytekitplugins-openai +flytekitplugins-pandera +flytekitplugins-papermill +flytekitplugins-polars +flytekitplugins-pydantic +flytekitplugins-ray +flytekitplugins-snowflake +flytekitplugins-spark +flytekitplugins-sqlalchemy +flytekitplugins-vaex +flytekitplugins-whylogs From 5bd8b27f0ad70395354665b980475500dcb45b29 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 08:19:05 +0800 Subject: [PATCH 10/47] Add more examples Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 4896900fb..e684032a1 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -26,6 +26,11 @@ jobs: - k8s_spark_plugin - athena_plugin - aws_batch_plugin + - basics + - bigquery_agent + - bigquery_plugin + - blast + - chatgpt_agent steps: - uses: actions/checkout@v4 - name: Set up Python From 95bf416f2aebd15a26ce595566d2f90a9a6e3a83 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 11:18:33 +0800 Subject: [PATCH 11/47] test Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index e684032a1..c3691e647 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -58,5 +58,6 @@ jobs: password: ${{ secrets.FLYTE_BOT_PAT }} - name: Pyflyte package run: | + source .venv/bin/activate cd examples/${{ matrix.example }} pyflyte --pkgs ${{ matrix.example }} package -o spark-package.tgz --force --fast From e42c5ba8ccacfa37028977335b2e1429b8f68e12 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 12:35:51 +0800 Subject: [PATCH 12/47] Update chatgpt example Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 14 ++++++++++++++ .../chatgpt_agent/chatgpt_agent_example_usage.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index c3691e647..c1734a784 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -31,6 +31,20 @@ jobs: - bigquery_plugin - blast - chatgpt_agent + - customizing_dependencies + - data_types_and_io + - dbt_plugin + - development_lifecycle + - dolt_plugin + - duckdb_plugin + - exploratory_data_analysis + - extending + - feast_integration + - forecasting_sales + - greatexpectations_plugin + - hive_plugin + - house_price_prediction + - k8s_dask_plugin steps: - uses: actions/checkout@v4 - name: Set up Python diff --git a/examples/chatgpt_agent/chatgpt_agent/chatgpt_agent_example_usage.py b/examples/chatgpt_agent/chatgpt_agent/chatgpt_agent_example_usage.py index 2f2702939..e778a9f64 100644 --- a/examples/chatgpt_agent/chatgpt_agent/chatgpt_agent_example_usage.py +++ b/examples/chatgpt_agent/chatgpt_agent/chatgpt_agent_example_usage.py @@ -133,7 +133,7 @@ def post_message_on_slack(message: str): def wf(owner: str = "flyteorg", repo: str = "flyte", channel: str = "demo"): message = get_github_latest_release(owner=owner, repo=repo) message = chatgpt_job(message=message) - post_message_on_slack(channel=channel, message=message) + post_message_on_slack(message=message) if __name__ == "__main__": From e1196b525c3fc11baf65a2d2712d79fa9958d06c Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 13:15:00 +0800 Subject: [PATCH 13/47] test Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 1 + requirements.in | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index c1734a784..33c4b2736 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -19,6 +19,7 @@ jobs: runs-on: "ubuntu-latest" name: Serialize strategy: + fail-fast: false matrix: example: - advanced_composition diff --git a/requirements.in b/requirements.in index 7e9e45195..73ccdaf13 100644 --- a/requirements.in +++ b/requirements.in @@ -34,3 +34,5 @@ flytekitplugins-spark flytekitplugins-sqlalchemy flytekitplugins-vaex flytekitplugins-whylogs +feast +horovod From 469640f025f0166a2c1e548d5aaf2bc4baf9837b Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 13:30:09 +0800 Subject: [PATCH 14/47] test Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 39 ++++++++++++-------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 33c4b2736..c5ddeb3c5 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -15,37 +15,25 @@ concurrency: cancel-in-progress: true jobs: + # This is the build system for the new example directory structure + list_examples: + runs-on: ubuntu-latest + name: "Create a list of example packages" + steps: + - uses: actions/checkout@v2 + - name: "Provide the list" + id: create-example-list + run: echo "PACKAGES=$(find examples -mindepth 1 -maxdepth 2 -type f -name Dockerfile -exec dirname '{}' \; | sort | sed 's/examples\///' | jq --raw-input . | jq --slurp . | jq -c .)" + outputs: + packages: "${{ steps.create-example-list.outputs.PACKAGES }}" serialize_examples: + needs: [list_examples] runs-on: "ubuntu-latest" name: Serialize strategy: fail-fast: false matrix: - example: - - advanced_composition - - airflow_agent - - k8s_spark_plugin - - athena_plugin - - aws_batch_plugin - - basics - - bigquery_agent - - bigquery_plugin - - blast - - chatgpt_agent - - customizing_dependencies - - data_types_and_io - - dbt_plugin - - development_lifecycle - - dolt_plugin - - duckdb_plugin - - exploratory_data_analysis - - extending - - feast_integration - - forecasting_sales - - greatexpectations_plugin - - hive_plugin - - house_price_prediction - - k8s_dask_plugin + example: "${{ fromJson(needs.list_examples.outputs.packages) }}" steps: - uses: actions/checkout@v4 - name: Set up Python @@ -61,6 +49,7 @@ jobs: key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }} - name: Install dependencies run: | + sudo apt install python3-packaging pip install uv uv venv source .venv/bin/activate From eaccdb60800f5e314d15f74472dd190bb1ed5217 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 13:38:26 +0800 Subject: [PATCH 15/47] test Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index c5ddeb3c5..189a6d06d 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/checkout@v2 - name: "Provide the list" id: create-example-list - run: echo "PACKAGES=$(find examples -mindepth 1 -maxdepth 2 -type f -name Dockerfile -exec dirname '{}' \; | sort | sed 's/examples\///' | jq --raw-input . | jq --slurp . | jq -c .)" + run: echo "PACKAGES=$(find examples -mindepth 1 -maxdepth 2 -type f -name Dockerfile -exec dirname '{}' \; | sort | sed 's/examples\///' | jq --raw-input . | jq --slurp . | jq -c .)" >> "$GITHUB_OUTPUT" outputs: packages: "${{ steps.create-example-list.outputs.PACKAGES }}" serialize_examples: From baccfdbc2c3c92e29d3b579e37eeb1b4970d7ca6 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 13:47:58 +0800 Subject: [PATCH 16/47] test Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 189a6d06d..46f8587dc 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/checkout@v2 - name: "Provide the list" id: create-example-list - run: echo "PACKAGES=$(find examples -mindepth 1 -maxdepth 2 -type f -name Dockerfile -exec dirname '{}' \; | sort | sed 's/examples\///' | jq --raw-input . | jq --slurp . | jq -c .)" >> "$GITHUB_OUTPUT" + run: echo "PACKAGES=$(find examples -maxdepth 1 -type d -exec basename '{}' \; | grep -v -e 'testing' | sort | jq --raw-input . | jq --slurp . | jq -c .)" >> "$GITHUB_OUTPUT" outputs: packages: "${{ steps.create-example-list.outputs.PACKAGES }}" serialize_examples: @@ -49,7 +49,7 @@ jobs: key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }} - name: Install dependencies run: | - sudo apt install python3-packaging + sudo apt-get update && sudo apt install python3-packaging pip install uv uv venv source .venv/bin/activate From 8a2c7e956674660352da8e1a29178d7347d29a40 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 13:49:49 +0800 Subject: [PATCH 17/47] test Signed-off-by: Kevin Su --- requirements.in | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.in b/requirements.in index 73ccdaf13..7caac68a3 100644 --- a/requirements.in +++ b/requirements.in @@ -36,3 +36,4 @@ flytekitplugins-vaex flytekitplugins-whylogs feast horovod +packaging \ No newline at end of file From 5f0634646430d2fc192649ac8757ee63caf50a36 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 13:57:31 +0800 Subject: [PATCH 18/47] test Signed-off-by: Kevin Su --- requirements.in | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements.in b/requirements.in index 7caac68a3..74f13e94b 100644 --- a/requirements.in +++ b/requirements.in @@ -35,5 +35,3 @@ flytekitplugins-sqlalchemy flytekitplugins-vaex flytekitplugins-whylogs feast -horovod -packaging \ No newline at end of file From f07c854b0b6a96c4687b54d8a56e530af3e5783a Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 14:03:43 +0800 Subject: [PATCH 19/47] test Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 10 +++++++++- dev-requirements.in | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 46f8587dc..8e0f5e9f2 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -23,12 +23,20 @@ jobs: - uses: actions/checkout@v2 - name: "Provide the list" id: create-example-list - run: echo "PACKAGES=$(find examples -maxdepth 1 -type d -exec basename '{}' \; | grep -v -e 'testing' | sort | jq --raw-input . | jq --slurp . | jq -c .)" >> "$GITHUB_OUTPUT" + run: | + echo "PACKAGES=$(find examples -maxdepth 1 -type d -exec basename '{}' \; \ + | grep -v -e 'testing' -e 'examples' \ + | sort \ + | jq --raw-input . \ + | jq --slurp . \ + | jq -c .)" >> "$GITHUB_OUTPUT" outputs: packages: "${{ steps.create-example-list.outputs.PACKAGES }}" serialize_examples: needs: [list_examples] runs-on: "ubuntu-latest" + env: + PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python name: Serialize strategy: fail-fast: false diff --git a/dev-requirements.in b/dev-requirements.in index d6048dfcc..d7f500fff 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -12,3 +12,5 @@ mock pytest mypy mashumaro +xgboost +dask[dataframe] From 90c48043eba7fb57d0822b293dc6f967f81efdd9 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 14:31:00 +0800 Subject: [PATCH 20/47] test Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 3 ++- requirements.in | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 8e0f5e9f2..3bc667c28 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -26,6 +26,7 @@ jobs: run: | echo "PACKAGES=$(find examples -maxdepth 1 -type d -exec basename '{}' \; \ | grep -v -e 'testing' -e 'examples' \ + | grep -v -e 'aiflow_plugin' -e 'forecasting_sales' \ | sort \ | jq --raw-input . \ | jq --slurp . \ @@ -72,4 +73,4 @@ jobs: run: | source .venv/bin/activate cd examples/${{ matrix.example }} - pyflyte --pkgs ${{ matrix.example }} package -o spark-package.tgz --force --fast + pyflyte -vvv --pkgs ${{ matrix.example }} package -o spark-package.tgz --force --fast diff --git a/requirements.in b/requirements.in index 74f13e94b..f8b7d9e62 100644 --- a/requirements.in +++ b/requirements.in @@ -35,3 +35,6 @@ flytekitplugins-sqlalchemy flytekitplugins-vaex flytekitplugins-whylogs feast +xgboost +modin +dask[dataframe] From 41085e96bcbdae741d36f000773b63e4029f2712 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 14:46:40 +0800 Subject: [PATCH 21/47] test Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 6 ++++-- requirements.in | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 3bc667c28..83ad85a53 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -26,7 +26,7 @@ jobs: run: | echo "PACKAGES=$(find examples -maxdepth 1 -type d -exec basename '{}' \; \ | grep -v -e 'testing' -e 'examples' \ - | grep -v -e 'aiflow_plugin' -e 'forecasting_sales' \ + | grep -v -e 'airflow_plugin' -e 'forecasting_sales' \ | sort \ | jq --raw-input . \ | jq --slurp . \ @@ -37,6 +37,7 @@ jobs: needs: [list_examples] runs-on: "ubuntu-latest" env: + FLYTE_SDK_RICH_TRACEBACKS: "0" PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python name: Serialize strategy: @@ -71,6 +72,7 @@ jobs: password: ${{ secrets.FLYTE_BOT_PAT }} - name: Pyflyte package run: | + pip list source .venv/bin/activate cd examples/${{ matrix.example }} - pyflyte -vvv --pkgs ${{ matrix.example }} package -o spark-package.tgz --force --fast + pyflyte -vvv --pkgs ${{ matrix.example }} package -o spark-package.tgz --force --fast diff --git a/requirements.in b/requirements.in index f8b7d9e62..0b1b0257c 100644 --- a/requirements.in +++ b/requirements.in @@ -38,3 +38,4 @@ feast xgboost modin dask[dataframe] +wandb From e3697d60ab7e5b286a48bf27b5433d31b96663d4 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 17:01:06 +0800 Subject: [PATCH 22/47] test Signed-off-by: Kevin Su --- examples/mmcloud_agent/requirements.txt | 358 ------------ examples/mmcloud_plugin/requirements.txt | 358 ------------ examples/mnist_classifier/requirements.txt | 231 -------- examples/modin_plugin/requirements.txt | 349 ------------ examples/nlp_processing/requirements.txt | 312 ---------- examples/pandera_plugin/requirements.txt | 532 ------------------ examples/papermill_plugin/requirements.txt | 331 ----------- examples/pima_diabetes/requirements.txt | 205 ------- examples/productionizing/requirements.in | 1 - examples/productionizing/requirements.txt | 345 ------------ examples/ray_plugin/requirements.txt | 429 -------------- .../requirements.txt | 367 ------------ examples/snowflake_agent/requirements.in | 3 +- examples/snowflake_agent/requirements.txt | 365 ------------ examples/snowflake_plugin/requirements.txt | 365 ------------ examples/sql_plugin/requirements.txt | 204 ------- examples/whylogs_plugin/requirements.txt | 358 ------------ 17 files changed, 2 insertions(+), 5111 deletions(-) delete mode 100644 examples/mmcloud_agent/requirements.txt delete mode 100644 examples/mmcloud_plugin/requirements.txt delete mode 100644 examples/mnist_classifier/requirements.txt delete mode 100644 examples/modin_plugin/requirements.txt delete mode 100644 examples/nlp_processing/requirements.txt delete mode 100644 examples/pandera_plugin/requirements.txt delete mode 100644 examples/papermill_plugin/requirements.txt delete mode 100644 examples/pima_diabetes/requirements.txt delete mode 100644 examples/productionizing/requirements.in delete mode 100644 examples/productionizing/requirements.txt delete mode 100644 examples/ray_plugin/requirements.txt delete mode 100644 examples/sagemaker_inference_agent/requirements.txt delete mode 100644 examples/snowflake_agent/requirements.txt delete mode 100644 examples/snowflake_plugin/requirements.txt delete mode 100644 examples/sql_plugin/requirements.txt delete mode 100644 examples/whylogs_plugin/requirements.txt diff --git a/examples/mmcloud_agent/requirements.txt b/examples/mmcloud_agent/requirements.txt deleted file mode 100644 index b0bb5e4e0..000000000 --- a/examples/mmcloud_agent/requirements.txt +++ /dev/null @@ -1,358 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# pip-compile requirements.in -# -adlfs==2023.9.0 - # via flytekit -aiobotocore==2.5.4 - # via s3fs -aiohttp==3.8.5 - # via - # adlfs - # aiobotocore - # gcsfs - # s3fs -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp -arrow==1.2.3 - # via cookiecutter -async-timeout==4.0.3 - # via aiohttp -attrs==23.1.0 - # via aiohttp -azure-core==1.29.4 - # via - # adlfs - # azure-identity - # azure-storage-blob -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.14.0 - # via adlfs -azure-storage-blob==12.18.2 - # via adlfs -binaryornot==0.4.4 - # via cookiecutter -botocore==1.31.17 - # via aiobotocore -cachetools==5.3.1 - # via google-auth -certifi==2023.7.22 - # via - # kubernetes - # requests -cffi==1.16.0 - # via - # azure-datalake-store - # cryptography -chardet==5.2.0 - # via binaryornot -charset-normalizer==3.2.0 - # via - # aiohttp - # requests -click==8.1.7 - # via - # cookiecutter - # flytekit - # rich-click -cloudpickle==2.2.1 - # via flytekit -cookiecutter==2.3.1 - # via flytekit -croniter==1.4.1 - # via flytekit -cryptography==41.0.6 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt - # pyopenssl - # secretstorage -dataclasses-json==0.5.9 - # via flytekit -decorator==5.1.1 - # via gcsfs -deprecated==1.2.14 - # via flytekit -diskcache==5.6.3 - # via flytekit -docker==6.1.3 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.15 - # via flytekit -envd==0.3.40 - # via flytekitplugins-envd -flyteidl==1.5.21 - # via flytekit -flytekit==1.9.1 - # via - # flytekitplugins-envd - # flytekitplugins-mmcloud -flytekitplugins-envd==1.9.1 - # via -r requirements.in -flytekitplugins-mmcloud==1.10.0b0 - # via -r requirements.in -frozenlist==1.4.0 - # via - # aiohttp - # aiosignal -fsspec==2023.9.2 - # via - # adlfs - # flytekit - # gcsfs - # s3fs -gcsfs==2023.9.2 - # via flytekit -gitdb==4.0.10 - # via gitpython -gitpython==3.1.37 - # via flytekit -google-api-core==2.12.0 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.23.2 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage - # kubernetes -google-auth-oauthlib==1.1.0 - # via gcsfs -google-cloud-core==2.3.3 - # via google-cloud-storage -google-cloud-storage==2.11.0 - # via gcsfs -google-crc32c==1.5.0 - # via google-resumable-media -google-resumable-media==2.6.0 - # via google-cloud-storage -googleapis-common-protos==1.60.0 - # via - # flyteidl - # flytekit - # google-api-core - # grpcio-status -grpcio==1.53.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.53.0 - # via flytekit -idna==3.4 - # via - # requests - # yarl -importlib-metadata==6.8.0 - # via - # flytekit - # keyring -isodate==0.6.1 - # via azure-storage-blob -jaraco-classes==3.3.0 - # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.2 - # via cookiecutter -jmespath==1.0.1 - # via botocore -joblib==1.3.2 - # via flytekit -jsonpickle==3.0.2 - # via flytekit -keyring==24.2.0 - # via flytekit -kubernetes==28.1.0 - # via flytekit -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.3 - # via jinja2 -marshmallow==3.20.1 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via - # dataclasses-json - # flytekit -marshmallow-jsonschema==0.13.0 - # via flytekit -mdurl==0.1.2 - # via markdown-it-py -more-itertools==10.1.0 - # via jaraco-classes -msal==1.24.0 - # via - # azure-datalake-store - # azure-identity - # msal-extensions -msal-extensions==1.0.0 - # via azure-identity -multidict==6.0.4 - # via - # aiohttp - # yarl -mypy-extensions==1.0.0 - # via typing-inspect -natsort==8.4.0 - # via flytekit -numpy==1.26.0 - # via - # flytekit - # pandas - # pyarrow -oauthlib==3.2.2 - # via - # kubernetes - # requests-oauthlib -packaging==23.1 - # via - # docker - # marshmallow -pandas==1.5.3 - # via flytekit -portalocker==2.8.2 - # via msal-extensions -protobuf==4.24.3 - # via - # flyteidl - # google-api-core - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -pyarrow==10.0.1 - # via flytekit -pyasn1==0.5.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pycparser==2.21 - # via cffi -pygments==2.16.1 - # via rich -pyjwt[crypto]==2.8.0 - # via - # msal - # pyjwt -pyopenssl==23.2.0 - # via flytekit -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # flytekit - # kubernetes - # pandas -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2023.3.post1 - # via - # flytekit - # pandas -pyyaml==6.0.1 - # via - # cookiecutter - # flytekit - # kubernetes -regex==2023.8.8 - # via docker-image-py -requests==2.31.0 - # via - # azure-core - # azure-datalake-store - # cookiecutter - # docker - # flytekit - # gcsfs - # google-api-core - # google-cloud-storage - # kubernetes - # msal - # requests-oauthlib -requests-oauthlib==1.3.1 - # via - # google-auth-oauthlib - # kubernetes -rich==13.5.3 - # via - # cookiecutter - # flytekit - # rich-click -rich-click==1.6.1 - # via flytekit -rsa==4.9 - # via google-auth -s3fs==2023.9.2 - # via flytekit -secretstorage==3.3.3 - # via keyring -six==1.16.0 - # via - # azure-core - # isodate - # kubernetes - # python-dateutil -smmap==5.0.1 - # via gitdb -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -typing-extensions==4.8.0 - # via - # aioitertools - # azure-core - # azure-storage-blob - # flytekit - # typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -urllib3==1.26.17 - # via - # botocore - # docker - # flytekit - # kubernetes - # requests -websocket-client==1.6.3 - # via - # docker - # kubernetes -wheel==0.41.2 - # via flytekit -wrapt==1.15.0 - # via - # aiobotocore - # deprecated - # flytekit -yarl==1.9.2 - # via aiohttp -zipp==3.17.0 - # via importlib-metadata diff --git a/examples/mmcloud_plugin/requirements.txt b/examples/mmcloud_plugin/requirements.txt deleted file mode 100644 index b0bb5e4e0..000000000 --- a/examples/mmcloud_plugin/requirements.txt +++ /dev/null @@ -1,358 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# pip-compile requirements.in -# -adlfs==2023.9.0 - # via flytekit -aiobotocore==2.5.4 - # via s3fs -aiohttp==3.8.5 - # via - # adlfs - # aiobotocore - # gcsfs - # s3fs -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp -arrow==1.2.3 - # via cookiecutter -async-timeout==4.0.3 - # via aiohttp -attrs==23.1.0 - # via aiohttp -azure-core==1.29.4 - # via - # adlfs - # azure-identity - # azure-storage-blob -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.14.0 - # via adlfs -azure-storage-blob==12.18.2 - # via adlfs -binaryornot==0.4.4 - # via cookiecutter -botocore==1.31.17 - # via aiobotocore -cachetools==5.3.1 - # via google-auth -certifi==2023.7.22 - # via - # kubernetes - # requests -cffi==1.16.0 - # via - # azure-datalake-store - # cryptography -chardet==5.2.0 - # via binaryornot -charset-normalizer==3.2.0 - # via - # aiohttp - # requests -click==8.1.7 - # via - # cookiecutter - # flytekit - # rich-click -cloudpickle==2.2.1 - # via flytekit -cookiecutter==2.3.1 - # via flytekit -croniter==1.4.1 - # via flytekit -cryptography==41.0.6 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt - # pyopenssl - # secretstorage -dataclasses-json==0.5.9 - # via flytekit -decorator==5.1.1 - # via gcsfs -deprecated==1.2.14 - # via flytekit -diskcache==5.6.3 - # via flytekit -docker==6.1.3 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.15 - # via flytekit -envd==0.3.40 - # via flytekitplugins-envd -flyteidl==1.5.21 - # via flytekit -flytekit==1.9.1 - # via - # flytekitplugins-envd - # flytekitplugins-mmcloud -flytekitplugins-envd==1.9.1 - # via -r requirements.in -flytekitplugins-mmcloud==1.10.0b0 - # via -r requirements.in -frozenlist==1.4.0 - # via - # aiohttp - # aiosignal -fsspec==2023.9.2 - # via - # adlfs - # flytekit - # gcsfs - # s3fs -gcsfs==2023.9.2 - # via flytekit -gitdb==4.0.10 - # via gitpython -gitpython==3.1.37 - # via flytekit -google-api-core==2.12.0 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.23.2 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage - # kubernetes -google-auth-oauthlib==1.1.0 - # via gcsfs -google-cloud-core==2.3.3 - # via google-cloud-storage -google-cloud-storage==2.11.0 - # via gcsfs -google-crc32c==1.5.0 - # via google-resumable-media -google-resumable-media==2.6.0 - # via google-cloud-storage -googleapis-common-protos==1.60.0 - # via - # flyteidl - # flytekit - # google-api-core - # grpcio-status -grpcio==1.53.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.53.0 - # via flytekit -idna==3.4 - # via - # requests - # yarl -importlib-metadata==6.8.0 - # via - # flytekit - # keyring -isodate==0.6.1 - # via azure-storage-blob -jaraco-classes==3.3.0 - # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.2 - # via cookiecutter -jmespath==1.0.1 - # via botocore -joblib==1.3.2 - # via flytekit -jsonpickle==3.0.2 - # via flytekit -keyring==24.2.0 - # via flytekit -kubernetes==28.1.0 - # via flytekit -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.3 - # via jinja2 -marshmallow==3.20.1 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via - # dataclasses-json - # flytekit -marshmallow-jsonschema==0.13.0 - # via flytekit -mdurl==0.1.2 - # via markdown-it-py -more-itertools==10.1.0 - # via jaraco-classes -msal==1.24.0 - # via - # azure-datalake-store - # azure-identity - # msal-extensions -msal-extensions==1.0.0 - # via azure-identity -multidict==6.0.4 - # via - # aiohttp - # yarl -mypy-extensions==1.0.0 - # via typing-inspect -natsort==8.4.0 - # via flytekit -numpy==1.26.0 - # via - # flytekit - # pandas - # pyarrow -oauthlib==3.2.2 - # via - # kubernetes - # requests-oauthlib -packaging==23.1 - # via - # docker - # marshmallow -pandas==1.5.3 - # via flytekit -portalocker==2.8.2 - # via msal-extensions -protobuf==4.24.3 - # via - # flyteidl - # google-api-core - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -pyarrow==10.0.1 - # via flytekit -pyasn1==0.5.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pycparser==2.21 - # via cffi -pygments==2.16.1 - # via rich -pyjwt[crypto]==2.8.0 - # via - # msal - # pyjwt -pyopenssl==23.2.0 - # via flytekit -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # flytekit - # kubernetes - # pandas -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2023.3.post1 - # via - # flytekit - # pandas -pyyaml==6.0.1 - # via - # cookiecutter - # flytekit - # kubernetes -regex==2023.8.8 - # via docker-image-py -requests==2.31.0 - # via - # azure-core - # azure-datalake-store - # cookiecutter - # docker - # flytekit - # gcsfs - # google-api-core - # google-cloud-storage - # kubernetes - # msal - # requests-oauthlib -requests-oauthlib==1.3.1 - # via - # google-auth-oauthlib - # kubernetes -rich==13.5.3 - # via - # cookiecutter - # flytekit - # rich-click -rich-click==1.6.1 - # via flytekit -rsa==4.9 - # via google-auth -s3fs==2023.9.2 - # via flytekit -secretstorage==3.3.3 - # via keyring -six==1.16.0 - # via - # azure-core - # isodate - # kubernetes - # python-dateutil -smmap==5.0.1 - # via gitdb -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -typing-extensions==4.8.0 - # via - # aioitertools - # azure-core - # azure-storage-blob - # flytekit - # typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -urllib3==1.26.17 - # via - # botocore - # docker - # flytekit - # kubernetes - # requests -websocket-client==1.6.3 - # via - # docker - # kubernetes -wheel==0.41.2 - # via flytekit -wrapt==1.15.0 - # via - # aiobotocore - # deprecated - # flytekit -yarl==1.9.2 - # via aiohttp -zipp==3.17.0 - # via importlib-metadata diff --git a/examples/mnist_classifier/requirements.txt b/examples/mnist_classifier/requirements.txt deleted file mode 100644 index 06fcafecf..000000000 --- a/examples/mnist_classifier/requirements.txt +++ /dev/null @@ -1,231 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# /Library/Developer/CommandLineTools/usr/bin/make requirements.txt -# -arrow==1.2.2 - # via jinja2-time -binaryornot==0.4.4 - # via cookiecutter -certifi==2021.10.8 - # via - # requests - # sentry-sdk -chardet==4.0.0 - # via binaryornot -charset-normalizer==2.0.12 - # via requests -checksumdir==1.2.0 - # via flytekit -click==8.1.2 - # via - # cookiecutter - # flytekit - # wandb -cloudpickle==2.0.0 - # via flytekit -cookiecutter==1.7.3 - # via flytekit -croniter==1.3.4 - # via flytekit -cycler==0.11.0 - # via matplotlib -dataclasses-json==0.5.7 - # via flytekit -decorator==5.1.1 - # via retry -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docker-pycreds==0.4.0 - # via wandb -docstring-parser==0.13 - # via flytekit -flyteidl==0.24.21 - # via flytekit -flytekit==0.32.6 - # via -r ../../../common/requirements-common.in -fonttools==4.33.2 - # via matplotlib -gitdb==4.0.9 - # via gitpython -gitpython==3.1.27 - # via wandb -googleapis-common-protos==1.56.0 - # via - # flyteidl - # grpcio-status -grpcio==1.44.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.44.0 - # via flytekit -idna==3.3 - # via requests -importlib-metadata==4.11.3 - # via keyring -jinja2==3.1.1 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 - # via cookiecutter -keyring==23.5.0 - # via flytekit -kiwisolver==1.4.2 - # via matplotlib -markupsafe==2.1.1 - # via jinja2 -marshmallow==3.15.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -matplotlib==3.5.1 - # via -r ../../../common/requirements-common.in -mypy-extensions==0.4.3 - # via typing-inspect -natsort==8.1.0 - # via flytekit -numpy==1.21.6 - # via - # -r requirements.in - # matplotlib - # pandas - # pyarrow - # torchvision -packaging==21.3 - # via - # marshmallow - # matplotlib -pandas==1.3.5 - # via - # -r requirements.in - # flytekit -pathtools==0.1.2 - # via wandb -pillow==9.1.0 - # via - # matplotlib - # torchvision -poyo==0.5.0 - # via cookiecutter -promise==2.3 - # via wandb -protobuf==3.20.1 - # via - # flyteidl - # flytekit - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger - # wandb -protoc-gen-swagger==0.1.0 - # via flyteidl -psutil==5.9.0 - # via wandb -py==1.11.0 - # via retry -pyarrow==6.0.1 - # via flytekit -pyparsing==3.0.8 - # via - # matplotlib - # packaging -python-dateutil==2.8.2 - # via - # croniter - # flytekit - # matplotlib - # pandas - # wandb -python-json-logger==2.0.2 - # via flytekit -python-slugify==6.1.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.1 - # via - # flytekit - # pandas -pyyaml==6.0 - # via - # flytekit - # wandb -regex==2022.4.24 - # via docker-image-py -requests==2.27.1 - # via - # cookiecutter - # flytekit - # responses - # torchvision - # wandb -responses==0.20.0 - # via flytekit -retry==0.9.2 - # via flytekit -sentry-sdk==1.5.10 - # via wandb -setproctitle==1.2.3 - # via wandb -shortuuid==1.0.8 - # via wandb -six==1.16.0 - # via - # cookiecutter - # docker-pycreds - # grpcio - # promise - # python-dateutil - # wandb -smmap==5.0.0 - # via gitdb -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -torch==1.11.0 - # via - # -r requirements.in - # torchvision -torchvision==0.12.0 - # via -r requirements.in -typing-extensions==4.2.0 - # via - # flytekit - # torch - # torchvision - # typing-inspect -typing-inspect==0.7.1 - # via dataclasses-json -urllib3==1.26.9 - # via - # flytekit - # requests - # responses - # sentry-sdk -wandb==0.12.15 - # via -r requirements.in -wheel==0.37.1 - # via - # -r ../../../common/requirements-common.in - # flytekit -wrapt==1.14.0 - # via - # deprecated - # flytekit -zipp==3.8.0 - # via importlib-metadata diff --git a/examples/modin_plugin/requirements.txt b/examples/modin_plugin/requirements.txt deleted file mode 100644 index 2418f13e8..000000000 --- a/examples/modin_plugin/requirements.txt +++ /dev/null @@ -1,349 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# /Library/Developer/CommandLineTools/usr/bin/make requirements.txt -# -aiosignal==1.2.0 - # via ray -arrow==1.2.3 - # via jinja2-time -attrs==22.1.0 - # via - # jsonschema - # ray - # visions -binaryornot==0.4.4 - # via cookiecutter -certifi==2022.9.24 - # via requests -cffi==1.15.1 - # via cryptography -chardet==5.0.0 - # via binaryornot -charset-normalizer==2.1.1 - # via requests -click==8.0.4 - # via - # cookiecutter - # flytekit - # ray -cloudpickle==2.2.0 - # via flytekit -cookiecutter==2.1.1 - # via flytekit -croniter==1.3.7 - # via flytekit -cryptography==38.0.3 - # via pyopenssl -cycler==0.11.0 - # via matplotlib -dataclasses-json==0.5.7 - # via flytekit -decorator==5.1.1 - # via retry -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -distlib==0.3.6 - # via virtualenv -docker==6.0.1 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.15 - # via flytekit -filelock==3.8.0 - # via - # ray - # virtualenv -flyteidl==1.1.22 - # via flytekit -flytekit==1.2.3 - # via - # -r ../../../common/requirements-common.in - # flytekitplugins-deck-standard - # flytekitplugins-modin -flytekitplugins-deck-standard==1.2.3 - # via -r ../../../common/requirements-common.in -flytekitplugins-modin==1.2.3 - # via -r requirements.in -fonttools==4.38.0 - # via matplotlib -frozenlist==1.3.1 - # via - # aiosignal - # ray -fsspec==2022.10.0 - # via - # flytekitplugins-modin - # modin -googleapis-common-protos==1.56.4 - # via - # flyteidl - # grpcio-status -grpcio==1.43.0 - # via - # flytekit - # flytekitplugins-modin - # grpcio-status - # ray -grpcio-status==1.43.0 - # via - # flytekit - # flytekitplugins-modin -htmlmin==0.1.12 - # via pandas-profiling -idna==3.4 - # via requests -imagehash==4.3.1 - # via visions -importlib-metadata==5.0.0 - # via - # flytekit - # keyring - # markdown -jaraco-classes==3.2.3 - # via keyring -jinja2==3.1.2 - # via - # cookiecutter - # jinja2-time - # pandas-profiling -jinja2-time==0.2.0 - # via cookiecutter -joblib==1.2.0 - # via - # flytekit - # phik - # scikit-learn -jsonschema==4.17.0 - # via ray -keyring==23.11.0 - # via flytekit -kiwisolver==1.4.4 - # via matplotlib -markdown==3.4.1 - # via flytekitplugins-deck-standard -markupsafe==2.1.1 - # via jinja2 -marshmallow==3.18.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -matplotlib==3.5.3 - # via - # -r ../../../common/requirements-common.in - # missingno - # pandas-profiling - # phik - # seaborn -missingno==0.5.1 - # via pandas-profiling -modin==0.16.2 - # via flytekitplugins-modin -more-itertools==9.0.0 - # via jaraco-classes -msgpack==1.0.4 - # via ray -multimethod==1.9 - # via - # pandas-profiling - # visions -mypy-extensions==0.4.3 - # via typing-inspect -natsort==8.2.0 - # via flytekit -networkx==2.8.8 - # via visions -numpy==1.23.4 - # via - # imagehash - # matplotlib - # missingno - # modin - # pandas - # pandas-profiling - # patsy - # phik - # pyarrow - # ray - # scikit-learn - # scipy - # seaborn - # statsmodels - # visions -packaging==21.3 - # via - # docker - # marshmallow - # matplotlib - # modin - # statsmodels -pandas==1.5.1 - # via - # flytekit - # modin - # pandas-profiling - # phik - # seaborn - # statsmodels - # visions -pandas-profiling==3.4.0 - # via flytekitplugins-deck-standard -patsy==0.5.3 - # via statsmodels -phik==0.12.2 - # via pandas-profiling -pillow==9.3.0 - # via - # imagehash - # matplotlib - # visions -platformdirs==2.5.3 - # via virtualenv -plotly==5.11.0 - # via flytekitplugins-deck-standard -protobuf==3.20.3 - # via - # flyteidl - # flytekit - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger - # ray -protoc-gen-swagger==0.1.0 - # via flyteidl -psutil==5.9.3 - # via modin -py==1.11.0 - # via retry -pyarrow==6.0.1 - # via flytekit -pycparser==2.21 - # via cffi -pydantic==1.10.2 - # via pandas-profiling -pyopenssl==22.1.0 - # via flytekit -pyparsing==3.0.9 - # via - # matplotlib - # packaging -pyrsistent==0.19.2 - # via jsonschema -python-dateutil==2.8.2 - # via - # arrow - # croniter - # flytekit - # matplotlib - # pandas -python-json-logger==2.0.4 - # via flytekit -python-slugify==6.1.2 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.6 - # via - # flytekit - # pandas -pywavelets==1.4.1 - # via imagehash -pyyaml==6.0 - # via - # cookiecutter - # flytekit - # pandas-profiling - # ray -ray==2.0.1 - # via flytekitplugins-modin -regex==2022.10.31 - # via docker-image-py -requests==2.28.1 - # via - # cookiecutter - # docker - # flytekit - # pandas-profiling - # ray - # responses -responses==0.22.0 - # via flytekit -retry==0.9.2 - # via flytekit -scikit-learn==1.1.3 - # via -r requirements.in -scipy==1.9.3 - # via - # imagehash - # missingno - # pandas-profiling - # phik - # scikit-learn - # statsmodels -seaborn==0.12.1 - # via - # missingno - # pandas-profiling -six==1.16.0 - # via - # grpcio - # patsy - # python-dateutil -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -statsmodels==0.13.5 - # via pandas-profiling -tangled-up-in-unicode==0.2.0 - # via visions -tenacity==8.1.0 - # via plotly -text-unidecode==1.3 - # via python-slugify -threadpoolctl==3.1.0 - # via scikit-learn -toml==0.10.2 - # via responses -tqdm==4.64.1 - # via pandas-profiling -types-toml==0.10.8 - # via responses -typing-extensions==4.4.0 - # via - # flytekit - # pydantic - # typing-inspect -typing-inspect==0.8.0 - # via dataclasses-json -urllib3==1.26.12 - # via - # docker - # flytekit - # requests - # responses -virtualenv==20.16.6 - # via ray -visions[type_image_path]==0.7.5 - # via pandas-profiling -websocket-client==1.4.2 - # via docker -wheel==0.38.2 - # via - # -r ../../../common/requirements-common.in - # flytekit -wrapt==1.14.1 - # via - # deprecated - # flytekit -zipp==3.10.0 - # via importlib-metadata diff --git a/examples/nlp_processing/requirements.txt b/examples/nlp_processing/requirements.txt deleted file mode 100644 index 219540c82..000000000 --- a/examples/nlp_processing/requirements.txt +++ /dev/null @@ -1,312 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# pip-compile requirements.in -# -arrow==1.2.2 - # via jinja2-time -attrs==22.1.0 - # via visions -binaryornot==0.4.4 - # via cookiecutter -certifi==2021.10.8 - # via requests -cffi==1.15.1 - # via cryptography -chardet==4.0.0 - # via binaryornot -charset-normalizer==2.0.12 - # via requests -click==8.1.2 - # via - # cookiecutter - # flytekit - # nltk -cloudpickle==2.0.0 - # via flytekit -cookiecutter==2.1.1 - # via flytekit -croniter==1.3.4 - # via flytekit -cryptography==38.0.1 - # via pyopenssl -cycler==0.11.0 - # via matplotlib -dataclasses-json==0.5.7 - # via flytekit -decorator==5.1.1 - # via retry -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -docker==5.0.3 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.13 - # via flytekit -flyteidl==1.1.14 - # via flytekit -flytekit==1.1.1 - # via - # -r ../../../common/requirements-common.in - # flytekitplugins-deck-standard -flytekitplugins-deck-standard==1.1.1 - # via -r ../../../common/requirements-common.in -fonttools==4.32.0 - # via matplotlib -gensim==4.2.0 - # via -r requirements.in -googleapis-common-protos==1.56.0 - # via - # flyteidl - # grpcio-status -grpcio==1.44.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.44.0 - # via flytekit -htmlmin==0.1.12 - # via pandas-profiling -idna==3.3 - # via requests -imagehash==4.3.1 - # via visions -importlib-metadata==4.11.3 - # via - # flytekit - # keyring - # markdown -jinja2==3.1.1 - # via - # cookiecutter - # jinja2-time - # pandas-profiling -jinja2-time==0.2.0 - # via cookiecutter -joblib==1.1.0 - # via - # nltk - # pandas-profiling - # phik - # scikit-learn -keyring==23.5.0 - # via flytekit -kiwisolver==1.4.2 - # via matplotlib -markdown==3.4.1 - # via flytekitplugins-deck-standard -markupsafe==2.1.1 - # via jinja2 -marshmallow==3.15.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -matplotlib==3.5.1 - # via - # -r ../../../common/requirements-common.in - # missingno - # pandas-profiling - # phik - # seaborn -missingno==0.5.1 - # via pandas-profiling -multimethod==1.8 - # via - # pandas-profiling - # visions -mypy-extensions==0.4.3 - # via typing-inspect -natsort==8.1.0 - # via flytekit -networkx==2.8.7 - # via visions -nltk==3.7 - # via -r requirements.in -numpy==1.22.3 - # via - # -r requirements.in - # gensim - # imagehash - # matplotlib - # missingno - # pandas - # pandas-profiling - # patsy - # phik - # pyarrow - # pyemd - # scikit-learn - # scipy - # seaborn - # statsmodels - # visions -packaging==21.3 - # via - # marshmallow - # matplotlib - # statsmodels -pandas==1.4.2 - # via - # flytekit - # pandas-profiling - # phik - # seaborn - # statsmodels - # visions -pandas-profiling==3.3.0 - # via flytekitplugins-deck-standard -patsy==0.5.3 - # via statsmodels -phik==0.12.2 - # via pandas-profiling -pillow==9.1.0 - # via - # imagehash - # matplotlib - # visions -plotly==5.10.0 - # via - # -r requirements.in - # flytekitplugins-deck-standard -protobuf==3.20.0 - # via - # flyteidl - # flytekit - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -py==1.11.0 - # via retry -pyarrow==6.0.1 - # via flytekit -pycparser==2.21 - # via cffi -pydantic==1.9.2 - # via pandas-profiling -pyemd==0.5.1 - # via -r requirements.in -pyopenssl==22.0.0 - # via flytekit -pyparsing==3.0.8 - # via - # matplotlib - # packaging -python-dateutil==2.8.2 - # via - # arrow - # croniter - # flytekit - # matplotlib - # pandas -python-json-logger==2.0.2 - # via flytekit -python-slugify==6.1.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.1 - # via - # flytekit - # pandas -pywavelets==1.3.0 - # via imagehash -pyyaml==6.0 - # via - # cookiecutter - # flytekit - # pandas-profiling -regex==2022.3.15 - # via - # docker-image-py - # nltk -requests==2.27.1 - # via - # cookiecutter - # docker - # flytekit - # pandas-profiling - # responses -responses==0.20.0 - # via flytekit -retry==0.9.2 - # via flytekit -scikit-learn==1.1.3 - # via -r requirements.in -scipy==1.8.0 - # via - # gensim - # imagehash - # missingno - # pandas-profiling - # phik - # scikit-learn - # seaborn - # statsmodels -seaborn==0.11.2 - # via - # missingno - # pandas-profiling -six==1.16.0 - # via - # grpcio - # patsy - # python-dateutil -smart-open==6.2.0 - # via gensim -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -statsmodels==0.13.2 - # via pandas-profiling -tangled-up-in-unicode==0.2.0 - # via - # pandas-profiling - # visions -tenacity==8.1.0 - # via plotly -text-unidecode==1.3 - # via python-slugify -threadpoolctl==3.1.0 - # via scikit-learn -tqdm==4.64.1 - # via - # nltk - # pandas-profiling -typing-extensions==4.1.1 - # via - # flytekit - # pydantic - # typing-inspect -typing-inspect==0.7.1 - # via dataclasses-json -urllib3==1.26.9 - # via - # flytekit - # requests - # responses -visions[type_image_path]==0.7.5 - # via pandas-profiling -websocket-client==1.4.1 - # via docker -wheel==0.37.1 - # via - # -r ../../../common/requirements-common.in - # flytekit -wrapt==1.14.0 - # via - # deprecated - # flytekit -zipp==3.8.0 - # via importlib-metadata diff --git a/examples/pandera_plugin/requirements.txt b/examples/pandera_plugin/requirements.txt deleted file mode 100644 index 850ddf858..000000000 --- a/examples/pandera_plugin/requirements.txt +++ /dev/null @@ -1,532 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile requirements.in -# -adlfs==2024.2.0 - # via flytekit -aiobotocore==2.12.2 - # via s3fs -aiohttp==3.9.3 - # via - # adlfs - # aiobotocore - # gcsfs - # s3fs -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp -annotated-types==0.6.0 - # via pydantic -arrow==1.3.0 - # via cookiecutter -asttokens==2.4.1 - # via stack-data -async-timeout==4.0.3 - # via aiohttp -attrs==23.2.0 - # via - # aiohttp - # hypothesis - # visions -azure-core==1.30.1 - # via - # adlfs - # azure-identity - # azure-storage-blob -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.15.0 - # via adlfs -azure-storage-blob==12.19.1 - # via adlfs -backports-tarfile==1.0.0 - # via jaraco-context -binaryornot==0.4.4 - # via cookiecutter -botocore==1.34.51 - # via aiobotocore -cachetools==5.3.3 - # via google-auth -certifi==2024.2.2 - # via requests -cffi==1.16.0 - # via - # azure-datalake-store - # cryptography -chardet==5.2.0 - # via binaryornot -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # cookiecutter - # flytekit - # rich-click -cloudpickle==3.0.0 - # via flytekit -comm==0.2.2 - # via ipywidgets -contourpy==1.2.1 - # via matplotlib -cookiecutter==2.6.0 - # via flytekit -croniter==2.0.3 - # via flytekit -cryptography==42.0.5 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt -cycler==0.12.1 - # via matplotlib -dacite==1.8.1 - # via ydata-profiling -dataclasses-json==0.5.9 - # via flytekit -decorator==5.1.1 - # via - # gcsfs - # ipython -diskcache==5.6.3 - # via flytekit -docker==6.1.3 - # via flytekit -docstring-parser==0.16 - # via flytekit -exceptiongroup==1.2.0 - # via - # hypothesis - # ipython -executing==2.0.1 - # via stack-data -flyteidl==1.11.1b1 - # via flytekit -flytekit==1.11.0 - # via - # -r requirements.in - # flytekitplugins-deck-standard - # flytekitplugins-pandera -flytekitplugins-deck-standard==1.11.0 - # via -r requirements.in -flytekitplugins-pandera==1.11.0 - # via -r requirements.in -fonttools==4.51.0 - # via matplotlib -frozenlist==1.4.1 - # via - # aiohttp - # aiosignal -fsspec==2024.3.1 - # via - # adlfs - # flytekit - # gcsfs - # s3fs -gcsfs==2024.3.1 - # via flytekit -google-api-core==2.18.0 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.29.0 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage -google-auth-oauthlib==1.2.0 - # via gcsfs -google-cloud-core==2.4.1 - # via google-cloud-storage -google-cloud-storage==2.16.0 - # via gcsfs -google-crc32c==1.5.0 - # via - # google-cloud-storage - # google-resumable-media -google-resumable-media==2.7.0 - # via google-cloud-storage -googleapis-common-protos==1.63.0 - # via - # flyteidl - # flytekit - # google-api-core - # grpcio-status - # protoc-gen-openapiv2 -grpcio==1.62.1 - # via - # flytekit - # grpcio-status -grpcio-status==1.62.1 - # via flytekit -htmlmin==0.1.12 - # via ydata-profiling -hypothesis==6.100.1 - # via -r requirements.in -idna==3.6 - # via - # requests - # yarl -imagehash==4.3.1 - # via - # visions - # ydata-profiling -importlib-metadata==7.1.0 - # via - # flytekit - # keyring -ipython==8.23.0 - # via ipywidgets -ipywidgets==8.1.2 - # via flytekitplugins-deck-standard -isodate==0.6.1 - # via - # azure-storage-blob - # flytekit -jaraco-classes==3.4.0 - # via keyring -jaraco-context==5.3.0 - # via keyring -jaraco-functools==4.0.0 - # via keyring -jedi==0.19.1 - # via ipython -jinja2==3.1.3 - # via - # cookiecutter - # ydata-profiling -jmespath==1.0.1 - # via botocore -joblib==1.4.0 - # via - # -r requirements.in - # flytekit - # phik - # scikit-learn -jsonpickle==3.0.3 - # via flytekit -jupyterlab-widgets==3.0.10 - # via ipywidgets -keyring==25.1.0 - # via flytekit -kiwisolver==1.4.5 - # via matplotlib -llvmlite==0.42.0 - # via numba -markdown==3.6 - # via flytekitplugins-deck-standard -markdown-it-py==3.0.0 - # via - # flytekit - # rich -markupsafe==2.1.5 - # via jinja2 -marshmallow==3.21.1 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via - # dataclasses-json - # flytekit -marshmallow-jsonschema==0.13.0 - # via flytekit -mashumaro==3.12 - # via flytekit -matplotlib==3.8.4 - # via - # -r requirements.in - # phik - # seaborn - # wordcloud - # ydata-profiling -matplotlib-inline==0.1.6 - # via ipython -mdurl==0.1.2 - # via markdown-it-py -more-itertools==10.2.0 - # via - # jaraco-classes - # jaraco-functools -msal==1.28.0 - # via - # azure-datalake-store - # azure-identity - # msal-extensions -msal-extensions==1.1.0 - # via azure-identity -multidict==6.0.5 - # via - # aiohttp - # yarl -multimethod==1.10 - # via - # pandera - # visions - # ydata-profiling -mypy-extensions==1.0.0 - # via typing-inspect -networkx==3.3 - # via visions -numba==0.59.1 - # via ydata-profiling -numpy==1.26.4 - # via - # contourpy - # imagehash - # matplotlib - # numba - # pandas - # pandera - # patsy - # phik - # pyarrow - # pywavelets - # scikit-learn - # scipy - # seaborn - # statsmodels - # visions - # wordcloud - # ydata-profiling -oauthlib==3.2.2 - # via requests-oauthlib -packaging==24.0 - # via - # docker - # marshmallow - # matplotlib - # msal-extensions - # pandera - # plotly - # statsmodels -pandas==2.2.1 - # via - # -r requirements.in - # flytekitplugins-deck-standard - # flytekitplugins-pandera - # pandera - # phik - # seaborn - # statsmodels - # visions - # ydata-profiling -pandera==0.18.3 - # via - # -r requirements.in - # flytekitplugins-pandera -parso==0.8.4 - # via jedi -patsy==0.5.6 - # via statsmodels -pexpect==4.9.0 - # via ipython -phik==0.12.4 - # via ydata-profiling -pillow==10.3.0 - # via - # imagehash - # matplotlib - # visions - # wordcloud -plotly==5.20.0 - # via flytekitplugins-deck-standard -portalocker==2.8.2 - # via msal-extensions -prompt-toolkit==3.0.43 - # via ipython -proto-plus==1.23.0 - # via google-api-core -protobuf==4.25.3 - # via - # flyteidl - # flytekit - # google-api-core - # googleapis-common-protos - # grpcio-status - # proto-plus - # protoc-gen-openapiv2 -protoc-gen-openapiv2==0.0.1 - # via flyteidl -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data -pyarrow==15.0.2 - # via flytekit -pyasn1==0.6.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.4.0 - # via google-auth -pycparser==2.22 - # via cffi -pydantic==2.6.4 - # via - # pandera - # ydata-profiling -pydantic-core==2.16.3 - # via pydantic -pygments==2.17.2 - # via - # flytekit - # flytekitplugins-deck-standard - # ipython - # rich -pyjwt[crypto]==2.8.0 - # via - # msal - # pyjwt -pyparsing==3.1.2 - # via matplotlib -python-dateutil==2.9.0.post0 - # via - # arrow - # botocore - # croniter - # matplotlib - # pandas -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.4 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2024.1 - # via - # croniter - # pandas -pywavelets==1.6.0 - # via imagehash -pyyaml==6.0.1 - # via - # cookiecutter - # flytekit - # ydata-profiling -requests==2.31.0 - # via - # azure-core - # azure-datalake-store - # cookiecutter - # docker - # flytekit - # gcsfs - # google-api-core - # google-cloud-storage - # msal - # requests-oauthlib - # ydata-profiling -requests-oauthlib==2.0.0 - # via google-auth-oauthlib -rich==13.7.1 - # via - # cookiecutter - # flytekit - # rich-click -rich-click==1.7.4 - # via flytekit -rsa==4.9 - # via google-auth -s3fs==2024.3.1 - # via flytekit -scikit-learn==1.4.1.post1 - # via -r requirements.in -scipy==1.11.4 - # via - # imagehash - # phik - # scikit-learn - # statsmodels - # ydata-profiling -seaborn==0.12.2 - # via ydata-profiling -six==1.16.0 - # via - # asttokens - # azure-core - # isodate - # patsy - # python-dateutil -sortedcontainers==2.4.0 - # via hypothesis -stack-data==0.6.3 - # via ipython -statsd==3.3.0 - # via flytekit -statsmodels==0.14.1 - # via ydata-profiling -tenacity==8.2.3 - # via plotly -text-unidecode==1.3 - # via python-slugify -threadpoolctl==3.4.0 - # via scikit-learn -tqdm==4.66.2 - # via ydata-profiling -traitlets==5.14.2 - # via - # comm - # ipython - # ipywidgets - # matplotlib-inline -typeguard==4.2.1 - # via - # pandera - # ydata-profiling -types-python-dateutil==2.9.0.20240316 - # via arrow -typing-extensions==4.11.0 - # via - # azure-core - # azure-storage-blob - # flytekit - # ipython - # mashumaro - # pydantic - # pydantic-core - # rich-click - # typeguard - # typing-inspect -typing-inspect==0.9.0 - # via - # dataclasses-json - # pandera -tzdata==2024.1 - # via pandas -urllib3==1.26.18 - # via - # botocore - # docker - # flytekit - # requests -visions[type-image-path]==0.7.6 - # via - # visions - # ydata-profiling -wcwidth==0.2.13 - # via prompt-toolkit -websocket-client==1.7.0 - # via docker -wheel==0.43.0 - # via -r requirements.in -widgetsnbextension==4.0.10 - # via ipywidgets -wordcloud==1.9.3 - # via ydata-profiling -wrapt==1.16.0 - # via - # aiobotocore - # pandera -yarl==1.9.4 - # via aiohttp -ydata-profiling==4.7.0 - # via flytekitplugins-deck-standard -zipp==3.18.1 - # via importlib-metadata diff --git a/examples/papermill_plugin/requirements.txt b/examples/papermill_plugin/requirements.txt deleted file mode 100644 index d60f54573..000000000 --- a/examples/papermill_plugin/requirements.txt +++ /dev/null @@ -1,331 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# /Library/Developer/CommandLineTools/usr/bin/make requirements.txt -# -ansiwrap==0.8.4 - # via papermill -appnope==0.1.3 - # via - # ipykernel - # ipython -arrow==1.2.2 - # via jinja2-time -asttokens==2.0.5 - # via stack-data -attrs==21.4.0 - # via jsonschema -backcall==0.2.0 - # via ipython -beautifulsoup4==4.11.1 - # via nbconvert -binaryornot==0.4.4 - # via cookiecutter -bleach==5.0.0 - # via nbconvert -certifi==2021.10.8 - # via requests -chardet==4.0.0 - # via binaryornot -charset-normalizer==2.0.12 - # via requests -checksumdir==1.2.0 - # via flytekit -click==8.1.2 - # via - # cookiecutter - # flytekit - # papermill -cloudpickle==2.0.0 - # via flytekit -cookiecutter==1.7.3 - # via flytekit -croniter==1.3.4 - # via flytekit -cycler==0.11.0 - # via matplotlib -dataclasses-json==0.5.7 - # via flytekit -debugpy==1.6.0 - # via ipykernel -decorator==5.1.1 - # via - # ipython - # retry -defusedxml==0.7.1 - # via nbconvert -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.13 - # via flytekit -entrypoints==0.4 - # via - # jupyter-client - # nbconvert - # papermill -executing==0.8.3 - # via stack-data -fastjsonschema==2.15.3 - # via nbformat -flyteidl==0.24.21 - # via flytekit -flytekit==0.32.3 - # via - # -r ../../../common/requirements-common.in - # flytekitplugins-papermill -flytekitplugins-papermill==0.32.3 - # via -r requirements.in -fonttools==4.32.0 - # via matplotlib -googleapis-common-protos==1.56.0 - # via - # flyteidl - # grpcio-status -grpcio==1.44.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.44.0 - # via flytekit -idna==3.3 - # via requests -importlib-metadata==4.11.3 - # via keyring -ipykernel==6.13.0 - # via flytekitplugins-papermill -ipython==8.2.0 - # via ipykernel -jedi==0.18.1 - # via ipython -jinja2==3.1.1 - # via - # cookiecutter - # jinja2-time - # nbconvert -jinja2-time==0.2.0 - # via cookiecutter -jsonschema==4.4.0 - # via nbformat -jupyter-client==7.2.2 - # via - # ipykernel - # nbclient -jupyter-core==4.9.2 - # via - # jupyter-client - # nbconvert - # nbformat -jupyterlab-pygments==0.2.1 - # via nbconvert -keyring==23.5.0 - # via flytekit -kiwisolver==1.4.2 - # via matplotlib -markupsafe==2.1.1 - # via - # jinja2 - # nbconvert -marshmallow==3.15.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -matplotlib==3.5.1 - # via -r ../../../common/requirements-common.in -matplotlib-inline==0.1.3 - # via - # ipykernel - # ipython -mistune==0.8.4 - # via nbconvert -mypy-extensions==0.4.3 - # via typing-inspect -natsort==8.1.0 - # via flytekit -nbclient==0.6.0 - # via - # nbconvert - # papermill -nbconvert==6.5.0 - # via flytekitplugins-papermill -nbformat==5.3.0 - # via - # nbclient - # nbconvert - # papermill -nest-asyncio==1.5.5 - # via - # ipykernel - # jupyter-client - # nbclient -numpy==1.22.3 - # via - # matplotlib - # pandas - # pyarrow -packaging==21.3 - # via - # ipykernel - # marshmallow - # matplotlib - # nbconvert -pandas==1.4.2 - # via flytekit -pandocfilters==1.5.0 - # via nbconvert -papermill==2.3.4 - # via flytekitplugins-papermill -parso==0.8.3 - # via jedi -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 - # via ipython -pillow==9.1.0 - # via matplotlib -poyo==0.5.0 - # via cookiecutter -prompt-toolkit==3.0.29 - # via ipython -protobuf==3.20.0 - # via - # flyteidl - # flytekit - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -psutil==5.9.0 - # via ipykernel -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data -py==1.11.0 - # via retry -pyarrow==6.0.1 - # via flytekit -pygments==2.11.2 - # via - # ipython - # nbconvert -pyparsing==3.0.8 - # via - # matplotlib - # packaging -pyrsistent==0.18.1 - # via jsonschema -python-dateutil==2.8.2 - # via - # arrow - # croniter - # flytekit - # jupyter-client - # matplotlib - # pandas -python-json-logger==2.0.2 - # via flytekit -python-slugify==6.1.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.1 - # via - # flytekit - # pandas -pyyaml==6.0 - # via - # flytekit - # papermill -pyzmq==22.3.0 - # via jupyter-client -regex==2022.3.15 - # via docker-image-py -requests==2.27.1 - # via - # cookiecutter - # flytekit - # papermill - # responses -responses==0.20.0 - # via flytekit -retry==0.9.2 - # via flytekit -six==1.16.0 - # via - # bleach - # cookiecutter - # grpcio - # python-dateutil -sortedcontainers==2.4.0 - # via flytekit -soupsieve==2.3.2 - # via beautifulsoup4 -stack-data==0.2.0 - # via ipython -statsd==3.3.0 - # via flytekit -tenacity==8.0.1 - # via papermill -text-unidecode==1.3 - # via python-slugify -textwrap3==0.9.2 - # via ansiwrap -tinycss2==1.1.1 - # via nbconvert -tornado==6.1 - # via - # ipykernel - # jupyter-client -tqdm==4.64.0 - # via papermill -traitlets==5.1.1 - # via - # ipykernel - # ipython - # jupyter-client - # jupyter-core - # matplotlib-inline - # nbclient - # nbconvert - # nbformat -typing-extensions==4.1.1 - # via - # flytekit - # typing-inspect -typing-inspect==0.7.1 - # via dataclasses-json -urllib3==1.26.9 - # via - # flytekit - # requests - # responses -wcwidth==0.2.5 - # via prompt-toolkit -webencodings==0.5.1 - # via - # bleach - # tinycss2 -wheel==0.37.1 - # via - # -r ../../../common/requirements-common.in - # flytekit -wrapt==1.14.0 - # via - # deprecated - # flytekit -zipp==3.8.0 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/examples/pima_diabetes/requirements.txt b/examples/pima_diabetes/requirements.txt deleted file mode 100644 index d1a80d985..000000000 --- a/examples/pima_diabetes/requirements.txt +++ /dev/null @@ -1,205 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# /Library/Developer/CommandLineTools/usr/bin/make requirements.txt -# -arrow==1.2.2 - # via jinja2-time -binaryornot==0.4.4 - # via cookiecutter -certifi==2021.10.8 - # via requests -chardet==4.0.0 - # via binaryornot -charset-normalizer==2.0.12 - # via requests -checksumdir==1.2.0 - # via flytekit -click==8.1.2 - # via - # cookiecutter - # flytekit -cloudpickle==2.0.0 - # via flytekit -cookiecutter==1.7.3 - # via flytekit -croniter==1.3.4 - # via flytekit -cycler==0.11.0 - # via matplotlib -dataclasses-json==0.5.7 - # via flytekit -decorator==5.1.1 - # via retry -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.13 - # via flytekit -flyteidl==0.24.21 - # via flytekit -flytekit==0.32.3 - # via -r ../../../common/requirements-common.in -fonttools==4.32.0 - # via matplotlib -googleapis-common-protos==1.56.0 - # via - # flyteidl - # grpcio-status -grpcio==1.44.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.44.0 - # via flytekit -idna==3.3 - # via requests -importlib-metadata==4.11.3 - # via keyring -jinja2==3.1.1 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 - # via cookiecutter -joblib==1.1.0 - # via - # -r requirements.in - # scikit-learn -keyring==23.5.0 - # via flytekit -kiwisolver==1.4.2 - # via matplotlib -markupsafe==2.1.1 - # via jinja2 -marshmallow==3.15.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -matplotlib==3.5.1 - # via - # -r ../../../common/requirements-common.in - # -r requirements.in -mypy-extensions==0.4.3 - # via typing-inspect -natsort==8.1.0 - # via flytekit -numpy==1.22.3 - # via - # matplotlib - # pandas - # pyarrow - # scikit-learn - # scipy - # xgboost -packaging==21.3 - # via - # marshmallow - # matplotlib -pandas==1.4.2 - # via flytekit -pillow==9.1.0 - # via matplotlib -poyo==0.5.0 - # via cookiecutter -protobuf==3.20.0 - # via - # flyteidl - # flytekit - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -py==1.11.0 - # via retry -pyarrow==6.0.1 - # via flytekit -pyparsing==3.0.8 - # via - # matplotlib - # packaging -python-dateutil==2.8.2 - # via - # croniter - # flytekit - # matplotlib - # pandas -python-json-logger==2.0.2 - # via flytekit -python-slugify==6.1.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.1 - # via - # flytekit - # pandas -pyyaml==6.0 - # via flytekit -regex==2022.3.15 - # via docker-image-py -requests==2.27.1 - # via - # cookiecutter - # flytekit - # responses -responses==0.20.0 - # via flytekit -retry==0.9.2 - # via flytekit -scikit-learn==1.0.2 - # via sklearn -scipy==1.8.0 - # via - # scikit-learn - # xgboost -six==1.16.0 - # via - # cookiecutter - # grpcio - # python-dateutil -sklearn==0.0 - # via -r requirements.in -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -tabulate==0.8.9 - # via -r requirements.in -text-unidecode==1.3 - # via python-slugify -threadpoolctl==3.1.0 - # via scikit-learn -typing-extensions==4.1.1 - # via - # flytekit - # typing-inspect -typing-inspect==0.7.1 - # via dataclasses-json -urllib3==1.26.9 - # via - # flytekit - # requests - # responses -wheel==0.37.1 - # via - # -r ../../../common/requirements-common.in - # flytekit -wrapt==1.14.0 - # via - # deprecated - # flytekit -xgboost==1.5.2 - # via -r requirements.in -zipp==3.8.0 - # via importlib-metadata diff --git a/examples/productionizing/requirements.in b/examples/productionizing/requirements.in deleted file mode 100644 index 3cfb26e38..000000000 --- a/examples/productionizing/requirements.in +++ /dev/null @@ -1 +0,0 @@ -flytekit diff --git a/examples/productionizing/requirements.txt b/examples/productionizing/requirements.txt deleted file mode 100644 index 0093dfbd0..000000000 --- a/examples/productionizing/requirements.txt +++ /dev/null @@ -1,345 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile requirements.in -# -adlfs==2023.8.0 - # via flytekit -aiobotocore==2.5.4 - # via s3fs -aiohttp==3.9.2 - # via - # adlfs - # aiobotocore - # gcsfs - # s3fs -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp -arrow==1.2.3 - # via cookiecutter -attrs==23.1.0 - # via aiohttp -azure-core==1.29.3 - # via - # adlfs - # azure-identity - # azure-storage-blob -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.14.0 - # via adlfs -azure-storage-blob==12.17.0 - # via adlfs -binaryornot==0.4.4 - # via cookiecutter -botocore==1.31.17 - # via aiobotocore -cachetools==5.3.1 - # via google-auth -certifi==2023.7.22 - # via - # kubernetes - # requests -cffi==1.15.1 - # via - # azure-datalake-store - # cryptography -chardet==5.2.0 - # via binaryornot -charset-normalizer==3.2.0 - # via requests -click==8.1.7 - # via - # cookiecutter - # flytekit - # rich-click -cloudpickle==2.2.1 - # via flytekit -cookiecutter==2.3.0 - # via flytekit -croniter==1.4.1 - # via flytekit -cryptography==41.0.3 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt - # pyopenssl - # secretstorage -dataclasses-json==0.5.9 - # via flytekit -decorator==5.1.1 - # via gcsfs -deprecated==1.2.14 - # via flytekit -diskcache==5.6.1 - # via flytekit -docker==6.1.3 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.15 - # via flytekit -flyteidl==1.5.17 - # via flytekit -flytekit==1.9.0 - # via -r requirements.in -frozenlist==1.4.0 - # via - # aiohttp - # aiosignal -fsspec==2023.6.0 - # via - # adlfs - # flytekit - # gcsfs - # s3fs -gcsfs==2023.6.0 - # via flytekit -gitdb==4.0.10 - # via gitpython -gitpython==3.1.32 - # via flytekit -google-api-core==2.11.1 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.22.0 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage - # kubernetes -google-auth-oauthlib==1.0.0 - # via gcsfs -google-cloud-core==2.3.3 - # via google-cloud-storage -google-cloud-storage==2.10.0 - # via gcsfs -google-crc32c==1.5.0 - # via google-resumable-media -google-resumable-media==2.5.0 - # via google-cloud-storage -googleapis-common-protos==1.60.0 - # via - # flyteidl - # flytekit - # google-api-core - # grpcio-status -grpcio==1.53.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.53.0 - # via flytekit -idna==3.4 - # via - # requests - # yarl -importlib-metadata==6.8.0 - # via - # flytekit - # keyring -isodate==0.6.1 - # via azure-storage-blob -jaraco-classes==3.3.0 - # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.2 - # via cookiecutter -jmespath==1.0.1 - # via botocore -joblib==1.3.2 - # via flytekit -keyring==24.2.0 - # via flytekit -kubernetes==27.2.0 - # via flytekit -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.3 - # via jinja2 -marshmallow==3.20.1 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via - # dataclasses-json - # flytekit -marshmallow-jsonschema==0.13.0 - # via flytekit -mdurl==0.1.2 - # via markdown-it-py -more-itertools==10.1.0 - # via jaraco-classes -msal==1.23.0 - # via - # azure-datalake-store - # azure-identity - # msal-extensions -msal-extensions==1.0.0 - # via azure-identity -multidict==6.0.4 - # via - # aiohttp - # yarl -mypy-extensions==1.0.0 - # via typing-inspect -natsort==8.4.0 - # via flytekit -numpy==1.25.2 - # via - # flytekit - # pandas - # pyarrow -oauthlib==3.2.2 - # via - # kubernetes - # requests-oauthlib -packaging==23.1 - # via - # docker - # marshmallow -pandas==1.5.3 - # via flytekit -portalocker==2.7.0 - # via msal-extensions -protobuf==4.24.2 - # via - # flyteidl - # google-api-core - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -pyarrow==10.0.1 - # via flytekit -pyasn1==0.5.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pycparser==2.21 - # via cffi -pygments==2.16.1 - # via rich -pyjwt[crypto]==2.8.0 - # via - # msal - # pyjwt -pyopenssl==23.2.0 - # via flytekit -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # flytekit - # kubernetes - # pandas -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2023.3 - # via - # flytekit - # pandas -pyyaml==6.0.1 - # via - # cookiecutter - # flytekit - # kubernetes -regex==2023.8.8 - # via docker-image-py -requests==2.31.0 - # via - # azure-core - # azure-datalake-store - # cookiecutter - # docker - # flytekit - # gcsfs - # google-api-core - # google-cloud-storage - # kubernetes - # msal - # requests-oauthlib -requests-oauthlib==1.3.1 - # via - # google-auth-oauthlib - # kubernetes -rich==13.5.2 - # via - # cookiecutter - # flytekit - # rich-click -rich-click==1.6.1 - # via flytekit -rsa==4.9 - # via google-auth -s3fs==2023.6.0 - # via flytekit -secretstorage==3.3.3 - # via keyring -six==1.16.0 - # via - # azure-core - # google-auth - # isodate - # kubernetes - # python-dateutil -smmap==5.0.0 - # via gitdb -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -typing-extensions==4.7.1 - # via - # azure-core - # azure-storage-blob - # flytekit - # typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -urllib3==1.26.16 - # via - # botocore - # docker - # flytekit - # google-auth - # kubernetes - # requests -websocket-client==1.6.2 - # via - # docker - # kubernetes -wheel==0.41.2 - # via flytekit -wrapt==1.15.0 - # via - # aiobotocore - # deprecated - # flytekit -yarl==1.9.2 - # via aiohttp -zipp==3.16.2 - # via importlib-metadata diff --git a/examples/ray_plugin/requirements.txt b/examples/ray_plugin/requirements.txt deleted file mode 100644 index a7daeb190..000000000 --- a/examples/ray_plugin/requirements.txt +++ /dev/null @@ -1,429 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile requirements.in -# -adlfs==2023.8.0 - # via flytekit -aiobotocore==2.5.4 - # via s3fs -aiohttp==3.8.5 - # via - # adlfs - # aiobotocore - # aiohttp-cors - # gcsfs - # ray - # s3fs -aiohttp-cors==0.7.0 - # via ray -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.3.1 - # via - # aiohttp - # ray -arrow==1.2.3 - # via cookiecutter -async-timeout==4.0.3 - # via aiohttp -attrs==23.1.0 - # via - # aiohttp - # jsonschema - # referencing -azure-core==1.29.2 - # via - # adlfs - # azure-identity - # azure-storage-blob -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.14.0 - # via adlfs -azure-storage-blob==12.17.0 - # via adlfs -binaryornot==0.4.4 - # via cookiecutter -blessed==1.20.0 - # via gpustat -botocore==1.31.17 - # via aiobotocore -cachetools==5.3.1 - # via google-auth -certifi==2023.7.22 - # via - # kubernetes - # requests -cffi==1.15.1 - # via - # azure-datalake-store - # cryptography -chardet==5.2.0 - # via binaryornot -charset-normalizer==3.2.0 - # via - # aiohttp - # requests -click==8.1.6 - # via - # cookiecutter - # flytekit - # ray - # rich-click -cloudpickle==2.2.1 - # via flytekit -colorful==0.5.5 - # via ray -cookiecutter==2.3.0 - # via flytekit -croniter==1.4.1 - # via flytekit -cryptography==41.0.6 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt - # pyopenssl - # secretstorage -dataclasses-json==0.5.9 - # via flytekit -decorator==5.1.1 - # via gcsfs -deprecated==1.2.14 - # via flytekit -diskcache==5.6.1 - # via flytekit -distlib==0.3.7 - # via virtualenv -docker==6.1.3 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.15 - # via flytekit -envd==0.3.38 - # via flytekitplugins-envd -filelock==3.12.2 - # via - # ray - # virtualenv -flyteidl==1.5.16 - # via - # flytekit - # flytekitplugins-ray -flytekit==1.9.0 - # via - # flytekitplugins-envd - # flytekitplugins-ray -flytekitplugins-envd==1.9.0 - # via -r requirements.in -flytekitplugins-ray==1.9.0 - # via -r requirements.in -frozenlist==1.4.0 - # via - # aiohttp - # aiosignal - # ray -fsspec==2023.6.0 - # via - # adlfs - # flytekit - # gcsfs - # s3fs -gcsfs==2023.6.0 - # via flytekit -gitdb==4.0.10 - # via gitpython -gitpython==3.1.32 - # via flytekit -google-api-core==2.11.1 - # via - # google-cloud-core - # google-cloud-storage - # opencensus -google-auth==2.22.0 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage - # kubernetes -google-auth-oauthlib==1.0.0 - # via gcsfs -google-cloud-core==2.3.3 - # via google-cloud-storage -google-cloud-storage==2.10.0 - # via gcsfs -google-crc32c==1.5.0 - # via google-resumable-media -google-resumable-media==2.5.0 - # via google-cloud-storage -googleapis-common-protos==1.60.0 - # via - # flyteidl - # flytekit - # google-api-core - # grpcio-status -gpustat==1.1 - # via ray -grpcio==1.53.0 - # via - # flytekit - # grpcio-status - # ray -grpcio-status==1.53.0 - # via flytekit -idna==3.4 - # via - # requests - # yarl -importlib-metadata==6.8.0 - # via - # flytekit - # keyring -isodate==0.6.1 - # via azure-storage-blob -jaraco-classes==3.3.0 - # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.2 - # via cookiecutter -jmespath==1.0.1 - # via botocore -joblib==1.3.2 - # via flytekit -jsonschema==4.19.0 - # via ray -jsonschema-specifications==2023.7.1 - # via jsonschema -keyring==24.2.0 - # via flytekit -kubernetes==27.2.0 - # via flytekit -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.3 - # via jinja2 -marshmallow==3.20.1 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via - # dataclasses-json - # flytekit -marshmallow-jsonschema==0.13.0 - # via flytekit -mdurl==0.1.2 - # via markdown-it-py -more-itertools==10.1.0 - # via jaraco-classes -msal==1.23.0 - # via - # azure-datalake-store - # azure-identity - # msal-extensions -msal-extensions==1.0.0 - # via azure-identity -msgpack==1.0.5 - # via ray -multidict==6.0.4 - # via - # aiohttp - # yarl -mypy-extensions==1.0.0 - # via typing-inspect -natsort==8.4.0 - # via flytekit -numpy==1.25.2 - # via - # flytekit - # pandas - # pyarrow - # ray -nvidia-ml-py==12.535.77 - # via gpustat -oauthlib==3.2.2 - # via - # kubernetes - # requests-oauthlib -opencensus==0.11.2 - # via ray -opencensus-context==0.1.3 - # via opencensus -packaging==23.1 - # via - # docker - # marshmallow - # ray -pandas==1.5.3 - # via flytekit -platformdirs==3.10.0 - # via virtualenv -portalocker==2.7.0 - # via msal-extensions -prometheus-client==0.17.1 - # via ray -protobuf==4.24.0 - # via - # flyteidl - # google-api-core - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger - # ray -protoc-gen-swagger==0.1.0 - # via flyteidl -psutil==5.9.5 - # via gpustat -py-spy==0.3.14 - # via ray -pyarrow==10.0.1 - # via flytekit -pyasn1==0.5.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pycparser==2.21 - # via cffi -pydantic==1.10.12 - # via ray -pygments==2.16.1 - # via rich -pyjwt[crypto]==2.8.0 - # via - # msal - # pyjwt -pyopenssl==23.2.0 - # via flytekit -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # flytekit - # kubernetes - # pandas -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2023.3 - # via - # flytekit - # pandas -pyyaml==6.0.1 - # via - # cookiecutter - # flytekit - # kubernetes - # ray -ray[default]==2.6.3 - # via flytekitplugins-ray -referencing==0.30.2 - # via - # jsonschema - # jsonschema-specifications -regex==2023.8.8 - # via docker-image-py -requests==2.31.0 - # via - # azure-core - # azure-datalake-store - # cookiecutter - # docker - # flytekit - # gcsfs - # google-api-core - # google-cloud-storage - # kubernetes - # msal - # ray - # requests-oauthlib -requests-oauthlib==1.3.1 - # via - # google-auth-oauthlib - # kubernetes -rich==13.5.2 - # via - # cookiecutter - # flytekit - # rich-click -rich-click==1.6.1 - # via flytekit -rpds-py==0.9.2 - # via - # jsonschema - # referencing -rsa==4.9 - # via google-auth -s3fs==2023.6.0 - # via flytekit -secretstorage==3.3.3 - # via keyring -six==1.16.0 - # via - # azure-core - # blessed - # google-auth - # isodate - # kubernetes - # python-dateutil -smart-open==6.3.0 - # via ray -smmap==5.0.0 - # via gitdb -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -typing-extensions==4.7.1 - # via - # azure-core - # azure-storage-blob - # flytekit - # pydantic - # typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -urllib3==1.26.16 - # via - # botocore - # docker - # flytekit - # google-auth - # kubernetes - # requests -virtualenv==20.21.0 - # via ray -wcwidth==0.2.6 - # via blessed -websocket-client==1.6.1 - # via - # docker - # kubernetes -wheel==0.41.1 - # via flytekit -wrapt==1.15.0 - # via - # aiobotocore - # deprecated - # flytekit -yarl==1.9.2 - # via aiohttp -zipp==3.16.2 - # via importlib-metadata diff --git a/examples/sagemaker_inference_agent/requirements.txt b/examples/sagemaker_inference_agent/requirements.txt deleted file mode 100644 index 13c332493..000000000 --- a/examples/sagemaker_inference_agent/requirements.txt +++ /dev/null @@ -1,367 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile requirements.in -# -adlfs==2024.2.0 - # via flytekit -aioboto3==12.3.0 - # via flytekitplugins-awssagemaker -aiobotocore[boto3]==2.11.2 - # via - # aioboto3 - # s3fs -aiohttp==3.9.3 - # via - # adlfs - # aiobotocore - # gcsfs - # s3fs -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp -annotated-types==0.6.0 - # via pydantic -anyio==4.3.0 - # via starlette -arrow==1.3.0 - # via cookiecutter -attrs==23.2.0 - # via aiohttp -azure-core==1.30.1 - # via - # adlfs - # azure-identity - # azure-storage-blob -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.15.0 - # via adlfs -azure-storage-blob==12.19.1 - # via adlfs -binaryornot==0.4.4 - # via cookiecutter -boto3==1.34.34 - # via aiobotocore -botocore==1.34.34 - # via - # aiobotocore - # boto3 - # s3transfer -cachetools==5.3.3 - # via google-auth -certifi==2024.2.2 - # via requests -cffi==1.16.0 - # via - # azure-datalake-store - # cryptography -chardet==5.2.0 - # via binaryornot -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # cookiecutter - # flytekit - # rich-click - # uvicorn -cloudpickle==3.0.0 - # via flytekit -cookiecutter==2.6.0 - # via flytekit -croniter==2.0.3 - # via flytekit -cryptography==42.0.5 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt -dataclasses-json==0.5.9 - # via flytekit -decorator==5.1.1 - # via gcsfs -diskcache==5.6.3 - # via flytekit -docker==6.1.3 - # via flytekit -docstring-parser==0.16 - # via flytekit -fastapi==0.110.0 - # via -r requirements.in -flyteidl==1.11.1b0 - # via flytekit -flytekit==1.11.0 - # via flytekitplugins-awssagemaker -flytekitplugins-awssagemaker==1.12.0b2 - # via -r requirements.in -frozenlist==1.4.1 - # via - # aiohttp - # aiosignal -fsspec==2024.3.1 - # via - # adlfs - # flytekit - # gcsfs - # s3fs -gcsfs==2024.3.1 - # via flytekit -google-api-core==2.18.0 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.29.0 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage -google-auth-oauthlib==1.2.0 - # via gcsfs -google-cloud-core==2.4.1 - # via google-cloud-storage -google-cloud-storage==2.16.0 - # via gcsfs -google-crc32c==1.5.0 - # via - # google-cloud-storage - # google-resumable-media -google-resumable-media==2.7.0 - # via google-cloud-storage -googleapis-common-protos==1.63.0 - # via - # flyteidl - # flytekit - # google-api-core - # grpcio-status - # protoc-gen-openapiv2 -grpcio==1.62.1 - # via - # flytekit - # grpcio-status -grpcio-status==1.62.1 - # via flytekit -h11==0.14.0 - # via uvicorn -idna==3.6 - # via - # anyio - # requests - # yarl -importlib-metadata==7.1.0 - # via flytekit -isodate==0.6.1 - # via - # azure-storage-blob - # flytekit -jaraco-classes==3.3.1 - # via keyring -jaraco-context==4.3.0 - # via keyring -jaraco-functools==4.0.0 - # via keyring -jinja2==3.1.3 - # via cookiecutter -jmespath==1.0.1 - # via - # boto3 - # botocore -joblib==1.3.2 - # via - # flytekit - # scikit-learn -jsonpickle==3.0.3 - # via flytekit -keyring==25.0.0 - # via flytekit -markdown-it-py==3.0.0 - # via - # flytekit - # rich -markupsafe==2.1.5 - # via jinja2 -marshmallow==3.21.1 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via - # dataclasses-json - # flytekit -marshmallow-jsonschema==0.13.0 - # via flytekit -mashumaro==3.12 - # via flytekit -mdurl==0.1.2 - # via markdown-it-py -more-itertools==10.2.0 - # via - # jaraco-classes - # jaraco-functools -msal==1.28.0 - # via - # azure-datalake-store - # azure-identity - # msal-extensions -msal-extensions==1.1.0 - # via azure-identity -multidict==6.0.5 - # via - # aiohttp - # yarl -mypy-extensions==1.0.0 - # via typing-inspect -numpy==1.26.4 - # via - # pyarrow - # scikit-learn - # scipy - # xgboost -oauthlib==3.2.2 - # via requests-oauthlib -packaging==24.0 - # via - # docker - # marshmallow - # msal-extensions -portalocker==2.8.2 - # via msal-extensions -proto-plus==1.23.0 - # via google-api-core -protobuf==4.25.3 - # via - # flyteidl - # flytekit - # google-api-core - # googleapis-common-protos - # grpcio-status - # proto-plus - # protoc-gen-openapiv2 -protoc-gen-openapiv2==0.0.1 - # via flyteidl -pyarrow==15.0.2 - # via flytekit -pyasn1==0.6.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.4.0 - # via google-auth -pycparser==2.21 - # via cffi -pydantic==2.6.4 - # via fastapi -pydantic-core==2.16.3 - # via pydantic -pygments==2.17.2 - # via - # flytekit - # rich -pyjwt[crypto]==2.8.0 - # via - # msal - # pyjwt -python-dateutil==2.9.0.post0 - # via - # arrow - # botocore - # croniter -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.4 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2024.1 - # via croniter -pyyaml==6.0.1 - # via - # cookiecutter - # flytekit -requests==2.31.0 - # via - # azure-core - # azure-datalake-store - # cookiecutter - # docker - # flytekit - # gcsfs - # google-api-core - # google-cloud-storage - # msal - # requests-oauthlib -requests-oauthlib==2.0.0 - # via google-auth-oauthlib -rich==13.7.1 - # via - # cookiecutter - # flytekit - # rich-click -rich-click==1.7.4 - # via flytekit -rsa==4.9 - # via google-auth -s3fs==2024.3.1 - # via flytekit -s3transfer==0.10.1 - # via boto3 -scikit-learn==1.4.1.post1 - # via -r requirements.in -scipy==1.12.0 - # via - # scikit-learn - # xgboost -six==1.16.0 - # via - # azure-core - # isodate - # python-dateutil -sniffio==1.3.1 - # via anyio -starlette==0.36.3 - # via fastapi -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -threadpoolctl==3.4.0 - # via scikit-learn -types-python-dateutil==2.9.0.20240316 - # via arrow -typing-extensions==4.10.0 - # via - # azure-core - # azure-storage-blob - # fastapi - # flytekit - # mashumaro - # pydantic - # pydantic-core - # rich-click - # typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -urllib3==1.26.18 - # via - # botocore - # docker - # flytekit - # requests -uvicorn==0.29.0 - # via -r requirements.in -websocket-client==1.7.0 - # via docker -wrapt==1.16.0 - # via aiobotocore -xgboost==2.0.3 - # via -r requirements.in -yarl==1.9.4 - # via aiohttp -zipp==3.18.1 - # via importlib-metadata diff --git a/examples/snowflake_agent/requirements.in b/examples/snowflake_agent/requirements.in index 029e7ddd5..20d2f99d4 100644 --- a/examples/snowflake_agent/requirements.in +++ b/examples/snowflake_agent/requirements.in @@ -1 +1,2 @@ -flytekitplugins-snowflake +flytekitplugins-snowflake==1.7.0 +flytekit==1.7.0 diff --git a/examples/snowflake_agent/requirements.txt b/examples/snowflake_agent/requirements.txt deleted file mode 100644 index 3863198ca..000000000 --- a/examples/snowflake_agent/requirements.txt +++ /dev/null @@ -1,365 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.8 -# by the following command: -# -# pip-compile --resolver=backtracking requirements.in -# -adlfs==2023.4.0 - # via flytekit -aiobotocore==2.5.1 - # via s3fs -aiohttp==3.9.2 - # via - # adlfs - # aiobotocore - # gcsfs - # s3fs -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp -arrow==1.2.3 - # via jinja2-time -async-timeout==4.0.2 - # via aiohttp -attrs==23.1.0 - # via aiohttp -azure-core==1.27.1 - # via - # adlfs - # azure-identity - # azure-storage-blob -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.13.0 - # via adlfs -azure-storage-blob==12.16.0 - # via adlfs -binaryornot==0.4.4 - # via cookiecutter -botocore==1.29.161 - # via aiobotocore -cachetools==5.3.1 - # via google-auth -certifi==2023.5.7 - # via - # kubernetes - # requests -cffi==1.15.1 - # via - # azure-datalake-store - # cryptography -chardet==5.1.0 - # via binaryornot -charset-normalizer==3.1.0 - # via requests -click==8.1.3 - # via - # cookiecutter - # flytekit - # rich-click -cloudpickle==2.2.1 - # via flytekit -cookiecutter==2.1.1 - # via flytekit -croniter==1.4.1 - # via flytekit -cryptography==41.0.1 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt - # pyopenssl - # secretstorage -dataclasses-json==0.5.9 - # via flytekit -decorator==5.1.1 - # via gcsfs -deprecated==1.2.14 - # via flytekit -diskcache==5.6.1 - # via flytekit -docker==6.1.3 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.15 - # via flytekit -flyteidl==1.5.12 - # via flytekit -flytekit==1.7.1b1 - # via flytekitplugins-snowflake -flytekitplugins-snowflake==1.7.0 - # via -r requirements.in -frozenlist==1.3.3 - # via - # aiohttp - # aiosignal -fsspec==2023.6.0 - # via - # adlfs - # flytekit - # gcsfs - # s3fs -gcsfs==2023.6.0 - # via flytekit -gitdb==4.0.10 - # via gitpython -gitpython==3.1.37 - # via flytekit -google-api-core==2.11.1 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.21.0 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage - # kubernetes -google-auth-oauthlib==1.0.0 - # via gcsfs -google-cloud-core==2.3.3 - # via google-cloud-storage -google-cloud-storage==2.10.0 - # via gcsfs -google-crc32c==1.5.0 - # via google-resumable-media -google-resumable-media==2.5.0 - # via google-cloud-storage -googleapis-common-protos==1.59.1 - # via - # flyteidl - # flytekit - # google-api-core - # grpcio-status -grpcio==1.56.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.56.0 - # via flytekit -idna==3.4 - # via - # requests - # yarl -importlib-metadata==6.7.0 - # via - # flytekit - # keyring -importlib-resources==6.1.1 - # via keyring -isodate==0.6.1 - # via azure-storage-blob -jaraco-classes==3.2.3 - # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.3 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 - # via cookiecutter -jmespath==1.0.1 - # via botocore -joblib==1.3.1 - # via flytekit -keyring==24.2.0 - # via flytekit -kubernetes==26.1.0 - # via flytekit -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.3 - # via jinja2 -marshmallow==3.19.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -mdurl==0.1.2 - # via markdown-it-py -more-itertools==9.1.0 - # via jaraco-classes -msal==1.22.0 - # via - # azure-datalake-store - # azure-identity - # msal-extensions -msal-extensions==1.0.0 - # via azure-identity -multidict==6.0.4 - # via - # aiohttp - # yarl -mypy-extensions==1.0.0 - # via typing-inspect -natsort==8.4.0 - # via flytekit -numpy==1.24.4 - # via - # flytekit - # pandas - # pyarrow -oauthlib==3.2.2 - # via requests-oauthlib -packaging==23.1 - # via - # docker - # marshmallow -pandas==1.5.3 - # via flytekit -portalocker==2.7.0 - # via msal-extensions -protobuf==4.23.3 - # via - # flyteidl - # google-api-core - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -pyarrow==10.0.1 - # via flytekit -pyasn1==0.5.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pycparser==2.21 - # via cffi -pygments==2.15.1 - # via rich -pyjwt[crypto]==2.7.0 - # via - # msal - # pyjwt -pyopenssl==23.2.0 - # via flytekit -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # flytekit - # kubernetes - # pandas -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2023.3 - # via - # flytekit - # pandas -pyyaml==6.0 - # via - # cookiecutter - # flytekit - # kubernetes - # responses -regex==2023.6.3 - # via docker-image-py -requests==2.31.0 - # via - # azure-core - # azure-datalake-store - # cookiecutter - # docker - # flytekit - # gcsfs - # google-api-core - # google-cloud-storage - # kubernetes - # msal - # requests-oauthlib - # responses -requests-oauthlib==1.3.1 - # via - # google-auth-oauthlib - # kubernetes -responses==0.23.1 - # via flytekit -rich==13.4.2 - # via - # flytekit - # rich-click -rich-click==1.6.1 - # via flytekit -rsa==4.9 - # via google-auth -s3fs==2023.6.0 - # via flytekit -secretstorage==3.3.3 - # via keyring -six==1.16.0 - # via - # azure-core - # azure-identity - # google-auth - # isodate - # kubernetes - # python-dateutil -smmap==5.0.0 - # via gitdb -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -types-pyyaml==6.0.12.10 - # via responses -typing-extensions==4.7.1 - # via - # aioitertools - # azure-core - # azure-storage-blob - # flytekit - # rich - # typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -urllib3==1.26.16 - # via - # botocore - # docker - # flytekit - # google-auth - # kubernetes - # requests - # responses -websocket-client==1.6.1 - # via - # docker - # kubernetes -wheel==0.40.0 - # via flytekit -wrapt==1.15.0 - # via - # aiobotocore - # deprecated - # flytekit -yarl==1.9.2 - # via aiohttp -zipp==3.15.0 - # via - # importlib-metadata - # importlib-resources - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/examples/snowflake_plugin/requirements.txt b/examples/snowflake_plugin/requirements.txt deleted file mode 100644 index 3863198ca..000000000 --- a/examples/snowflake_plugin/requirements.txt +++ /dev/null @@ -1,365 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.8 -# by the following command: -# -# pip-compile --resolver=backtracking requirements.in -# -adlfs==2023.4.0 - # via flytekit -aiobotocore==2.5.1 - # via s3fs -aiohttp==3.9.2 - # via - # adlfs - # aiobotocore - # gcsfs - # s3fs -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp -arrow==1.2.3 - # via jinja2-time -async-timeout==4.0.2 - # via aiohttp -attrs==23.1.0 - # via aiohttp -azure-core==1.27.1 - # via - # adlfs - # azure-identity - # azure-storage-blob -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.13.0 - # via adlfs -azure-storage-blob==12.16.0 - # via adlfs -binaryornot==0.4.4 - # via cookiecutter -botocore==1.29.161 - # via aiobotocore -cachetools==5.3.1 - # via google-auth -certifi==2023.5.7 - # via - # kubernetes - # requests -cffi==1.15.1 - # via - # azure-datalake-store - # cryptography -chardet==5.1.0 - # via binaryornot -charset-normalizer==3.1.0 - # via requests -click==8.1.3 - # via - # cookiecutter - # flytekit - # rich-click -cloudpickle==2.2.1 - # via flytekit -cookiecutter==2.1.1 - # via flytekit -croniter==1.4.1 - # via flytekit -cryptography==41.0.1 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt - # pyopenssl - # secretstorage -dataclasses-json==0.5.9 - # via flytekit -decorator==5.1.1 - # via gcsfs -deprecated==1.2.14 - # via flytekit -diskcache==5.6.1 - # via flytekit -docker==6.1.3 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.15 - # via flytekit -flyteidl==1.5.12 - # via flytekit -flytekit==1.7.1b1 - # via flytekitplugins-snowflake -flytekitplugins-snowflake==1.7.0 - # via -r requirements.in -frozenlist==1.3.3 - # via - # aiohttp - # aiosignal -fsspec==2023.6.0 - # via - # adlfs - # flytekit - # gcsfs - # s3fs -gcsfs==2023.6.0 - # via flytekit -gitdb==4.0.10 - # via gitpython -gitpython==3.1.37 - # via flytekit -google-api-core==2.11.1 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.21.0 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage - # kubernetes -google-auth-oauthlib==1.0.0 - # via gcsfs -google-cloud-core==2.3.3 - # via google-cloud-storage -google-cloud-storage==2.10.0 - # via gcsfs -google-crc32c==1.5.0 - # via google-resumable-media -google-resumable-media==2.5.0 - # via google-cloud-storage -googleapis-common-protos==1.59.1 - # via - # flyteidl - # flytekit - # google-api-core - # grpcio-status -grpcio==1.56.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.56.0 - # via flytekit -idna==3.4 - # via - # requests - # yarl -importlib-metadata==6.7.0 - # via - # flytekit - # keyring -importlib-resources==6.1.1 - # via keyring -isodate==0.6.1 - # via azure-storage-blob -jaraco-classes==3.2.3 - # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.3 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 - # via cookiecutter -jmespath==1.0.1 - # via botocore -joblib==1.3.1 - # via flytekit -keyring==24.2.0 - # via flytekit -kubernetes==26.1.0 - # via flytekit -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.3 - # via jinja2 -marshmallow==3.19.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -mdurl==0.1.2 - # via markdown-it-py -more-itertools==9.1.0 - # via jaraco-classes -msal==1.22.0 - # via - # azure-datalake-store - # azure-identity - # msal-extensions -msal-extensions==1.0.0 - # via azure-identity -multidict==6.0.4 - # via - # aiohttp - # yarl -mypy-extensions==1.0.0 - # via typing-inspect -natsort==8.4.0 - # via flytekit -numpy==1.24.4 - # via - # flytekit - # pandas - # pyarrow -oauthlib==3.2.2 - # via requests-oauthlib -packaging==23.1 - # via - # docker - # marshmallow -pandas==1.5.3 - # via flytekit -portalocker==2.7.0 - # via msal-extensions -protobuf==4.23.3 - # via - # flyteidl - # google-api-core - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -pyarrow==10.0.1 - # via flytekit -pyasn1==0.5.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pycparser==2.21 - # via cffi -pygments==2.15.1 - # via rich -pyjwt[crypto]==2.7.0 - # via - # msal - # pyjwt -pyopenssl==23.2.0 - # via flytekit -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # flytekit - # kubernetes - # pandas -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2023.3 - # via - # flytekit - # pandas -pyyaml==6.0 - # via - # cookiecutter - # flytekit - # kubernetes - # responses -regex==2023.6.3 - # via docker-image-py -requests==2.31.0 - # via - # azure-core - # azure-datalake-store - # cookiecutter - # docker - # flytekit - # gcsfs - # google-api-core - # google-cloud-storage - # kubernetes - # msal - # requests-oauthlib - # responses -requests-oauthlib==1.3.1 - # via - # google-auth-oauthlib - # kubernetes -responses==0.23.1 - # via flytekit -rich==13.4.2 - # via - # flytekit - # rich-click -rich-click==1.6.1 - # via flytekit -rsa==4.9 - # via google-auth -s3fs==2023.6.0 - # via flytekit -secretstorage==3.3.3 - # via keyring -six==1.16.0 - # via - # azure-core - # azure-identity - # google-auth - # isodate - # kubernetes - # python-dateutil -smmap==5.0.0 - # via gitdb -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -types-pyyaml==6.0.12.10 - # via responses -typing-extensions==4.7.1 - # via - # aioitertools - # azure-core - # azure-storage-blob - # flytekit - # rich - # typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -urllib3==1.26.16 - # via - # botocore - # docker - # flytekit - # google-auth - # kubernetes - # requests - # responses -websocket-client==1.6.1 - # via - # docker - # kubernetes -wheel==0.40.0 - # via flytekit -wrapt==1.15.0 - # via - # aiobotocore - # deprecated - # flytekit -yarl==1.9.2 - # via aiohttp -zipp==3.15.0 - # via - # importlib-metadata - # importlib-resources - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/examples/sql_plugin/requirements.txt b/examples/sql_plugin/requirements.txt deleted file mode 100644 index cc1dec0a8..000000000 --- a/examples/sql_plugin/requirements.txt +++ /dev/null @@ -1,204 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# /Library/Developer/CommandLineTools/usr/bin/make requirements.txt -# -arrow==1.2.2 - # via jinja2-time -binaryornot==0.4.4 - # via cookiecutter -certifi==2021.10.8 - # via requests -cffi==1.15.0 - # via cryptography -chardet==4.0.0 - # via binaryornot -charset-normalizer==2.0.12 - # via requests -checksumdir==1.2.0 - # via flytekit -click==8.1.2 - # via - # cookiecutter - # flytekit -cloudpickle==2.0.0 - # via flytekit -cookiecutter==2.1.1 - # via flytekit -croniter==1.3.4 - # via flytekit -cryptography==37.0.3 - # via secretstorage -cycler==0.11.0 - # via matplotlib -dataclasses-json==0.5.7 - # via flytekit -decorator==5.1.1 - # via retry -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.13 - # via flytekit -flyteidl==0.24.21 - # via flytekit -flytekit==0.32.3 - # via - # -r ../../../common/requirements-common.in - # flytekitplugins-sqlalchemy -flytekitplugins-sqlalchemy==0.32.3 - # via -r requirements.in -fonttools==4.32.0 - # via matplotlib -googleapis-common-protos==1.56.0 - # via - # flyteidl - # grpcio-status -greenlet==1.1.2 - # via sqlalchemy -grpcio==1.44.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.44.0 - # via flytekit -idna==3.3 - # via requests -importlib-metadata==4.11.3 - # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.1 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 - # via cookiecutter -keyring==23.5.0 - # via flytekit -kiwisolver==1.4.2 - # via matplotlib -markupsafe==2.1.1 - # via jinja2 -marshmallow==3.15.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -matplotlib==3.5.1 - # via -r ../../../common/requirements-common.in -mypy-extensions==0.4.3 - # via typing-inspect -natsort==8.1.0 - # via flytekit -numpy==1.22.3 - # via - # matplotlib - # pandas - # pyarrow -packaging==21.3 - # via - # marshmallow - # matplotlib -pandas==1.4.2 - # via flytekit -pillow==9.1.0 - # via matplotlib -protobuf==3.20.0 - # via - # flyteidl - # flytekit - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -psycopg2-binary==2.9.3 - # via -r requirements.in -py==1.11.0 - # via retry -pyarrow==6.0.1 - # via flytekit -pycparser==2.21 - # via cffi -pyparsing==3.0.8 - # via - # matplotlib - # packaging -python-dateutil==2.8.2 - # via - # arrow - # croniter - # flytekit - # matplotlib - # pandas -python-json-logger==2.0.2 - # via flytekit -python-slugify==6.1.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.1 - # via - # flytekit - # pandas -pyyaml==6.0 - # via - # cookiecutter - # flytekit -regex==2022.3.15 - # via docker-image-py -requests==2.27.1 - # via - # cookiecutter - # flytekit - # responses -responses==0.20.0 - # via flytekit -retry==0.9.2 - # via flytekit -secretstorage==3.3.2 - # via keyring -six==1.16.0 - # via - # grpcio - # python-dateutil -sortedcontainers==2.4.0 - # via flytekit -sqlalchemy==1.4.35 - # via flytekitplugins-sqlalchemy -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -typing-extensions==4.1.1 - # via - # flytekit - # typing-inspect -typing-inspect==0.7.1 - # via dataclasses-json -urllib3==1.26.9 - # via - # flytekit - # requests - # responses -wheel==0.37.1 - # via - # -r ../../../common/requirements-common.in - # flytekit -wrapt==1.14.0 - # via - # deprecated - # flytekit -zipp==3.8.0 - # via importlib-metadata diff --git a/examples/whylogs_plugin/requirements.txt b/examples/whylogs_plugin/requirements.txt deleted file mode 100644 index bab22361c..000000000 --- a/examples/whylogs_plugin/requirements.txt +++ /dev/null @@ -1,358 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# pip-compile requirements.in -# -arrow==1.2.2 - # via jinja2-time -attrs==22.1.0 - # via visions -binaryornot==0.4.4 - # via cookiecutter -boto3==1.24.45 - # via whylogs -botocore==1.27.45 - # via - # boto3 - # s3transfer -certifi==2022.6.15 - # via requests -cffi==1.15.1 - # via cryptography -chardet==5.0.0 - # via binaryornot -charset-normalizer==2.1.0 - # via requests -click==8.1.3 - # via - # cookiecutter - # databricks-cli - # flytekit - # mlflow-skinny -cloudpickle==2.1.0 - # via - # flytekit - # mlflow-skinny -cookiecutter==2.1.1 - # via flytekit -croniter==1.3.5 - # via flytekit -cryptography==37.0.4 - # via - # pyopenssl - # secretstorage -cycler==0.11.0 - # via matplotlib -databricks-cli==0.17.0 - # via mlflow-skinny -dataclasses-json==0.5.7 - # via flytekit -decorator==5.1.1 - # via retry -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -docker==5.0.3 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.14.1 - # via flytekit -entrypoints==0.4 - # via mlflow-skinny -flake8==4.0.1 - # via whylogs -flyteidl==1.1.9 - # via flytekit -flytekit==1.1.0 - # via - # -r ../../../common/requirements-common.in - # flytekitplugins-deck-standard -flytekitplugins-deck-standard==1.1.0 - # via -r ../../../common/requirements-common.in -flytekitplugins-whylogs==1.1.1b0 - # via -r requirements.in -fonttools==4.34.4 - # via matplotlib -gitdb==4.0.9 - # via gitpython -gitpython==3.1.27 - # via mlflow-skinny -googleapis-common-protos==1.56.4 - # via - # flyteidl - # grpcio-status -grpcio==1.47.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.47.0 - # via flytekit -htmlmin==0.1.12 - # via pandas-profiling -idna==3.3 - # via requests -imagehash==4.2.1 - # via visions -importlib-metadata==4.12.0 - # via - # flytekit - # mlflow-skinny -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.2 - # via - # cookiecutter - # jinja2-time - # pandas-profiling -jinja2-time==0.2.0 - # via cookiecutter -jmespath==1.0.1 - # via - # boto3 - # botocore -joblib==1.1.0 - # via - # pandas-profiling - # phik - # scikit-learn -keyring==23.7.0 - # via flytekit -kiwisolver==1.4.4 - # via matplotlib -markdown==3.4.1 - # via flytekitplugins-deck-standard -markupsafe==2.1.1 - # via - # jinja2 - # pandas-profiling -marshmallow==3.17.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -matplotlib==3.5.2 - # via - # -r ../../../common/requirements-common.in - # missingno - # pandas-profiling - # phik - # seaborn -mccabe==0.6.1 - # via flake8 -missingno==0.5.1 - # via pandas-profiling -mlflow-skinny==1.27.0 - # via whylogs -multimethod==1.8 - # via - # pandas-profiling - # visions -mypy-extensions==0.4.3 - # via typing-inspect -natsort==8.1.0 - # via flytekit -networkx==2.8.5 - # via visions -numpy==1.23.1 - # via - # imagehash - # matplotlib - # missingno - # pandas - # pandas-profiling - # phik - # pyarrow - # scikit-learn - # scipy - # seaborn - # visions -oauthlib==3.2.1 - # via databricks-cli -packaging==21.3 - # via - # marshmallow - # matplotlib - # mlflow-skinny -pandas==1.4.3 - # via - # flytekit - # pandas-profiling - # phik - # seaborn - # visions -pandas-profiling==3.2.0 - # via flytekitplugins-deck-standard -phik==0.12.2 - # via pandas-profiling -pillow==9.2.0 - # via - # imagehash - # matplotlib - # visions -plotly==5.9.0 - # via flytekitplugins-deck-standard -protobuf==3.20.1 - # via - # flyteidl - # flytekit - # flytekitplugins-whylogs - # googleapis-common-protos - # grpcio-status - # mlflow-skinny - # protoc-gen-swagger - # whylogs -protoc-gen-swagger==0.1.0 - # via flyteidl -py==1.11.0 - # via retry -pyarrow==6.0.1 - # via flytekit -pycodestyle==2.8.0 - # via flake8 -pycparser==2.21 - # via cffi -pydantic==1.9.1 - # via pandas-profiling -pyflakes==2.4.0 - # via flake8 -pyjwt==2.4.0 - # via databricks-cli -pyopenssl==22.0.0 - # via flytekit -pyparsing==3.0.9 - # via - # matplotlib - # packaging -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # flytekit - # matplotlib - # pandas - # whylabs-client -python-json-logger==2.0.4 - # via flytekit -python-slugify==6.1.2 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.1 - # via - # flytekit - # mlflow-skinny - # pandas -pywavelets==1.3.0 - # via imagehash -pyyaml==6.0 - # via - # cookiecutter - # flytekit - # mlflow-skinny - # pandas-profiling -regex==2022.7.25 - # via docker-image-py -requests==2.28.1 - # via - # cookiecutter - # databricks-cli - # docker - # flytekit - # mlflow-skinny - # pandas-profiling - # responses -responses==0.21.0 - # via flytekit -retry==0.9.2 - # via flytekit -s3transfer==0.6.0 - # via boto3 -scikit-learn==1.1.2 - # via -r requirements.in -scipy==1.9.0 - # via - # imagehash - # missingno - # pandas-profiling - # phik - # scikit-learn - # seaborn -seaborn==0.11.2 - # via - # missingno - # pandas-profiling -secretstorage==3.3.3 - # via keyring -six==1.16.0 - # via - # databricks-cli - # grpcio - # imagehash - # python-dateutil -smmap==5.0.0 - # via gitdb -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -tabulate==0.8.10 - # via databricks-cli -tangled-up-in-unicode==0.2.0 - # via - # pandas-profiling - # visions -tenacity==8.0.1 - # via plotly -text-unidecode==1.3 - # via python-slugify -threadpoolctl==3.1.0 - # via scikit-learn -tqdm==4.64.0 - # via pandas-profiling -typing-extensions==4.3.0 - # via - # flytekit - # pydantic - # typing-inspect - # whylogs -typing-inspect==0.7.1 - # via dataclasses-json -urllib3==1.26.11 - # via - # botocore - # flytekit - # requests - # responses - # whylabs-client -visions[type_image_path]==0.7.4 - # via pandas-profiling -websocket-client==1.3.3 - # via docker -wheel==0.37.1 - # via - # -r ../../../common/requirements-common.in - # flytekit -whylabs-client==0.3.0 - # via whylogs -whylogs[mlflow,s3,whylabs]==1.0.9 - # via - # -r requirements.in - # flytekitplugins-whylogs -whylogs-sketching==3.4.1.dev2 - # via whylogs -wrapt==1.14.1 - # via - # deprecated - # flytekit -zipp==3.8.1 - # via importlib-metadata From 82a3587660b9b7fdb3ea62f2714b229c7041e250 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 17:06:14 +0800 Subject: [PATCH 23/47] test Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 11 ++++++----- examples/mmcloud_agent/requirements.in | 1 - examples/mmcloud_plugin/requirements.in | 1 - examples/ray_plugin/requirements.in | 1 - examples/snowflake_plugin/requirements.in | 3 ++- 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 83ad85a53..9348a5a42 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -26,7 +26,7 @@ jobs: run: | echo "PACKAGES=$(find examples -maxdepth 1 -type d -exec basename '{}' \; \ | grep -v -e 'testing' -e 'examples' \ - | grep -v -e 'airflow_plugin' -e 'forecasting_sales' \ + | grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' \ | sort \ | jq --raw-input . \ | jq --slurp . \ @@ -58,12 +58,13 @@ jobs: # Look to see if there is a cache hit for the corresponding requirements files key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }} - name: Install dependencies + working-directory: ${{ matrix.example }} run: | - sudo apt-get update && sudo apt install python3-packaging pip install uv uv venv source .venv/bin/activate - uv pip install -r requirements.in + uv pip install flytekit flytekitplugins-envd + if [ -f requirements.in ]; then uv pip install -r requirements.in; fi - name: Login to GitHub Container Registry uses: docker/login-action@v1 with: @@ -71,8 +72,8 @@ jobs: username: ${{ secrets.FLYTE_BOT_USERNAME }} password: ${{ secrets.FLYTE_BOT_PAT }} - name: Pyflyte package + working-directory: ${{ matrix.example }} run: | - pip list + pip freeze source .venv/bin/activate - cd examples/${{ matrix.example }} pyflyte -vvv --pkgs ${{ matrix.example }} package -o spark-package.tgz --force --fast diff --git a/examples/mmcloud_agent/requirements.in b/examples/mmcloud_agent/requirements.in index b490acb14..17c327692 100644 --- a/examples/mmcloud_agent/requirements.in +++ b/examples/mmcloud_agent/requirements.in @@ -1,2 +1 @@ -flytekitplugins-envd flytekitplugins-mmcloud diff --git a/examples/mmcloud_plugin/requirements.in b/examples/mmcloud_plugin/requirements.in index b490acb14..17c327692 100644 --- a/examples/mmcloud_plugin/requirements.in +++ b/examples/mmcloud_plugin/requirements.in @@ -1,2 +1 @@ -flytekitplugins-envd flytekitplugins-mmcloud diff --git a/examples/ray_plugin/requirements.in b/examples/ray_plugin/requirements.in index 512867625..5928c1284 100644 --- a/examples/ray_plugin/requirements.in +++ b/examples/ray_plugin/requirements.in @@ -1,2 +1 @@ flytekitplugins-ray -flytekitplugins-envd diff --git a/examples/snowflake_plugin/requirements.in b/examples/snowflake_plugin/requirements.in index 029e7ddd5..20d2f99d4 100644 --- a/examples/snowflake_plugin/requirements.in +++ b/examples/snowflake_plugin/requirements.in @@ -1 +1,2 @@ -flytekitplugins-snowflake +flytekitplugins-snowflake==1.7.0 +flytekit==1.7.0 From 5380b6acb26116856c1753f4bb75659a3e078e1f Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 17:10:00 +0800 Subject: [PATCH 24/47] test Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 9348a5a42..80ec7bffb 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -58,7 +58,7 @@ jobs: # Look to see if there is a cache hit for the corresponding requirements files key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }} - name: Install dependencies - working-directory: ${{ matrix.example }} + working-directory: examples/${{ matrix.example }} run: | pip install uv uv venv @@ -72,7 +72,7 @@ jobs: username: ${{ secrets.FLYTE_BOT_USERNAME }} password: ${{ secrets.FLYTE_BOT_PAT }} - name: Pyflyte package - working-directory: ${{ matrix.example }} + working-directory: examples/${{ matrix.example }} run: | pip freeze source .venv/bin/activate From c7c0f8942805cc824e272bd96ee5b403e6741a25 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 17:21:38 +0800 Subject: [PATCH 25/47] test Signed-off-by: Kevin Su --- examples/airflow_agent/requirements.in | 1 + examples/chatgpt_agent/requirements.in | 1 + .../customizing_dependencies/requirements.in | 2 +- .../customizing_dependencies/requirements.txt | 361 ------------- examples/data_types_and_io/requirements.in | 1 + examples/feast_integration/requirements.in | 1 + examples/feast_integration/requirements.txt | 500 ------------------ examples/hive_plugin/requirements.in | 1 + .../{requirements.txt => requirements.in} | 1 - examples/kfmpi_plugin/requirements.in | 2 +- examples/kfmpi_plugin/requirements.txt | 355 ------------- examples/kfpytorch_plugin/requirements.in | 2 +- examples/kfpytorch_plugin/requirements.txt | 349 ------------ examples/kftensorflow_plugin/requirements.in | 2 +- examples/kftensorflow_plugin/requirements.txt | 396 -------------- .../pytorch_single_node_and_gpu.py | 8 +- examples/modin_plugin/requirements.in | 1 + .../sagemaker_inference_agent/requirements.in | 2 + examples/snowflake_agent/requirements.in | 2 +- examples/snowflake_plugin/requirements.in | 2 +- 20 files changed, 16 insertions(+), 1974 deletions(-) create mode 100644 examples/airflow_agent/requirements.in create mode 100644 examples/chatgpt_agent/requirements.in delete mode 100644 examples/customizing_dependencies/requirements.txt create mode 100644 examples/data_types_and_io/requirements.in delete mode 100644 examples/feast_integration/requirements.txt create mode 100644 examples/hive_plugin/requirements.in rename examples/k8s_spark_plugin/{requirements.txt => requirements.in} (58%) delete mode 100644 examples/kfmpi_plugin/requirements.txt delete mode 100644 examples/kfpytorch_plugin/requirements.txt delete mode 100644 examples/kftensorflow_plugin/requirements.txt diff --git a/examples/airflow_agent/requirements.in b/examples/airflow_agent/requirements.in new file mode 100644 index 000000000..c1907c6cb --- /dev/null +++ b/examples/airflow_agent/requirements.in @@ -0,0 +1 @@ +flytekitplugins-airflow diff --git a/examples/chatgpt_agent/requirements.in b/examples/chatgpt_agent/requirements.in new file mode 100644 index 000000000..28e9bf61f --- /dev/null +++ b/examples/chatgpt_agent/requirements.in @@ -0,0 +1 @@ +flytekitplugins-openai \ No newline at end of file diff --git a/examples/customizing_dependencies/requirements.in b/examples/customizing_dependencies/requirements.in index f5303d5a4..8705f3c93 100644 --- a/examples/customizing_dependencies/requirements.in +++ b/examples/customizing_dependencies/requirements.in @@ -1,3 +1,3 @@ -flytekit +flytekit==1.9.0 flytekitplugins-envd scikit-learn diff --git a/examples/customizing_dependencies/requirements.txt b/examples/customizing_dependencies/requirements.txt deleted file mode 100644 index 1d1417ce7..000000000 --- a/examples/customizing_dependencies/requirements.txt +++ /dev/null @@ -1,361 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile requirements.in -# -adlfs==2023.8.0 - # via flytekit -aiobotocore==2.5.4 - # via s3fs -aiohttp==3.9.2 - # via - # adlfs - # aiobotocore - # gcsfs - # s3fs -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp -arrow==1.2.3 - # via cookiecutter -attrs==23.1.0 - # via aiohttp -azure-core==1.29.2 - # via - # adlfs - # azure-identity - # azure-storage-blob -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.14.0 - # via adlfs -azure-storage-blob==12.17.0 - # via adlfs -binaryornot==0.4.4 - # via cookiecutter -botocore==1.31.17 - # via aiobotocore -cachetools==5.3.1 - # via google-auth -certifi==2023.7.22 - # via - # kubernetes - # requests -cffi==1.15.1 - # via - # azure-datalake-store - # cryptography -chardet==5.2.0 - # via binaryornot -charset-normalizer==3.2.0 - # via requests -click==8.1.7 - # via - # cookiecutter - # flytekit - # rich-click -cloudpickle==2.2.1 - # via flytekit -cookiecutter==2.3.0 - # via flytekit -croniter==1.4.1 - # via flytekit -cryptography==41.0.6 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt - # pyopenssl - # secretstorage -dataclasses-json==0.5.9 - # via flytekit -decorator==5.1.1 - # via gcsfs -deprecated==1.2.14 - # via flytekit -diskcache==5.6.1 - # via flytekit -docker==6.1.3 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.15 - # via flytekit -envd==0.3.39 - # via flytekitplugins-envd -flyteidl==1.5.16 - # via flytekit -flytekit==1.9.0 - # via - # -r requirements.in - # flytekitplugins-envd -flytekitplugins-envd==1.9.0 - # via -r requirements.in -frozenlist==1.4.0 - # via - # aiohttp - # aiosignal -fsspec==2023.6.0 - # via - # adlfs - # flytekit - # gcsfs - # s3fs -gcsfs==2023.6.0 - # via flytekit -gitdb==4.0.10 - # via gitpython -gitpython==3.1.32 - # via flytekit -google-api-core==2.11.1 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.22.0 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage - # kubernetes -google-auth-oauthlib==1.0.0 - # via gcsfs -google-cloud-core==2.3.3 - # via google-cloud-storage -google-cloud-storage==2.10.0 - # via gcsfs -google-crc32c==1.5.0 - # via google-resumable-media -google-resumable-media==2.5.0 - # via google-cloud-storage -googleapis-common-protos==1.60.0 - # via - # flyteidl - # flytekit - # google-api-core - # grpcio-status -grpcio==1.53.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.53.0 - # via flytekit -idna==3.4 - # via - # requests - # yarl -importlib-metadata==6.8.0 - # via - # flytekit - # keyring -isodate==0.6.1 - # via azure-storage-blob -jaraco-classes==3.3.0 - # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.2 - # via cookiecutter -jmespath==1.0.1 - # via botocore -joblib==1.3.2 - # via - # flytekit - # scikit-learn -keyring==24.2.0 - # via flytekit -kubernetes==27.2.0 - # via flytekit -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.3 - # via jinja2 -marshmallow==3.20.1 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via - # dataclasses-json - # flytekit -marshmallow-jsonschema==0.13.0 - # via flytekit -mdurl==0.1.2 - # via markdown-it-py -more-itertools==10.1.0 - # via jaraco-classes -msal==1.23.0 - # via - # azure-datalake-store - # azure-identity - # msal-extensions -msal-extensions==1.0.0 - # via azure-identity -multidict==6.0.4 - # via - # aiohttp - # yarl -mypy-extensions==1.0.0 - # via typing-inspect -natsort==8.4.0 - # via flytekit -numpy==1.25.2 - # via - # flytekit - # pandas - # pyarrow - # scikit-learn - # scipy -oauthlib==3.2.2 - # via - # kubernetes - # requests-oauthlib -packaging==23.1 - # via - # docker - # marshmallow -pandas==1.5.3 - # via flytekit -portalocker==2.7.0 - # via msal-extensions -protobuf==4.24.1 - # via - # flyteidl - # google-api-core - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -pyarrow==10.0.1 - # via flytekit -pyasn1==0.5.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pycparser==2.21 - # via cffi -pygments==2.16.1 - # via rich -pyjwt[crypto]==2.8.0 - # via - # msal - # pyjwt -pyopenssl==23.2.0 - # via flytekit -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # flytekit - # kubernetes - # pandas -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2023.3 - # via - # flytekit - # pandas -pyyaml==6.0.1 - # via - # cookiecutter - # flytekit - # kubernetes -regex==2023.8.8 - # via docker-image-py -requests==2.31.0 - # via - # azure-core - # azure-datalake-store - # cookiecutter - # docker - # flytekit - # gcsfs - # google-api-core - # google-cloud-storage - # kubernetes - # msal - # requests-oauthlib -requests-oauthlib==1.3.1 - # via - # google-auth-oauthlib - # kubernetes -rich==13.5.2 - # via - # cookiecutter - # flytekit - # rich-click -rich-click==1.6.1 - # via flytekit -rsa==4.9 - # via google-auth -s3fs==2023.6.0 - # via flytekit -scikit-learn==1.3.0 - # via -r requirements.in -scipy==1.11.2 - # via scikit-learn -secretstorage==3.3.3 - # via keyring -six==1.16.0 - # via - # azure-core - # google-auth - # isodate - # kubernetes - # python-dateutil -smmap==5.0.0 - # via gitdb -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -threadpoolctl==3.2.0 - # via scikit-learn -typing-extensions==4.7.1 - # via - # azure-core - # azure-storage-blob - # flytekit - # typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -urllib3==1.26.16 - # via - # botocore - # docker - # flytekit - # google-auth - # kubernetes - # requests -websocket-client==1.6.1 - # via - # docker - # kubernetes -wheel==0.41.1 - # via flytekit -wrapt==1.15.0 - # via - # aiobotocore - # deprecated - # flytekit -yarl==1.9.2 - # via aiohttp -zipp==3.16.2 - # via importlib-metadata diff --git a/examples/data_types_and_io/requirements.in b/examples/data_types_and_io/requirements.in new file mode 100644 index 000000000..1411a4a0b --- /dev/null +++ b/examples/data_types_and_io/requirements.in @@ -0,0 +1 @@ +pandas \ No newline at end of file diff --git a/examples/feast_integration/requirements.in b/examples/feast_integration/requirements.in index 8b5c6d71f..436da3159 100644 --- a/examples/feast_integration/requirements.in +++ b/examples/feast_integration/requirements.in @@ -5,3 +5,4 @@ flytekitplugins-deck-standard scikit-learn numpy feast[aws] +boto3 diff --git a/examples/feast_integration/requirements.txt b/examples/feast_integration/requirements.txt deleted file mode 100644 index 98dc42e49..000000000 --- a/examples/feast_integration/requirements.txt +++ /dev/null @@ -1,500 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# /Library/Developer/CommandLineTools/usr/bin/make requirements.txt -# -absl-py==1.2.0 - # via tensorflow-metadata -aiobotocore==2.1.2 - # via s3fs -aiohttp==3.8.1 - # via - # aiobotocore - # s3fs -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.2.0 - # via aiohttp -anyio==3.6.1 - # via - # starlette - # watchfiles -appdirs==1.4.4 - # via fissix -arrow==1.2.3 - # via jinja2-time -async-timeout==4.0.2 - # via aiohttp -attrs==22.1.0 - # via - # aiohttp - # bowler - # jsonschema - # visions -binaryornot==0.4.4 - # via cookiecutter -boto3==1.20.23 - # via feast -botocore==1.23.24 - # via - # aiobotocore - # boto3 - # s3transfer -bowler==0.9.0 - # via feast -cachetools==5.2.0 - # via google-auth -certifi==2022.9.14 - # via requests -cffi==1.15.1 - # via cryptography -chardet==5.0.0 - # via binaryornot -charset-normalizer==2.1.1 - # via - # aiohttp - # requests -click==8.1.3 - # via - # bowler - # cookiecutter - # feast - # flytekit - # moreorless - # uvicorn -cloudpickle==2.2.0 - # via - # dask - # flytekit -colorama==0.4.5 - # via feast -cookiecutter==2.1.1 - # via flytekit -croniter==1.3.7 - # via flytekit -cryptography==38.0.1 - # via pyopenssl -cycler==0.11.0 - # via matplotlib -dask==2022.1.1 - # via feast -dataclasses-json==0.5.7 - # via flytekit -decorator==5.1.1 - # via retry -deprecated==1.2.13 - # via flytekit -dill==0.3.5.1 - # via feast -diskcache==5.4.0 - # via flytekit -docker==6.0.0 - # via - # feast - # flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.15 - # via flytekit -fastapi==0.85.0 - # via feast -fastavro==1.6.1 - # via - # feast - # pandavro -feast[aws]==0.25.0 - # via -r requirements.in -fissix==21.11.13 - # via bowler -flyteidl==1.1.17 - # via flytekit -flytekit==1.1.3 - # via - # -r ../../../common/requirements-common.in - # flytekitplugins-deck-standard -flytekitplugins-deck-standard==1.1.3 - # via -r ../../../common/requirements-common.in -fonttools==4.37.3 - # via matplotlib -frozenlist==1.3.1 - # via - # aiohttp - # aiosignal -fsspec==2022.1.0 - # via - # dask - # s3fs -google-api-core==2.10.1 - # via feast -google-auth==2.11.1 - # via google-api-core -googleapis-common-protos==1.56.4 - # via - # feast - # flyteidl - # google-api-core - # grpcio-status - # tensorflow-metadata -greenlet==1.1.3 - # via sqlalchemy -grpcio==1.48.1 - # via - # feast - # flytekit - # grpcio-reflection - # grpcio-status -grpcio-reflection==1.48.1 - # via feast -grpcio-status==1.48.1 - # via flytekit -h11==0.13.0 - # via uvicorn -htmlmin==0.1.12 - # via pandas-profiling -httptools==0.5.0 - # via uvicorn -idna==3.4 - # via - # anyio - # requests - # yarl -imagehash==4.3.0 - # via visions -importlib-metadata==4.12.0 - # via - # flytekit - # keyring - # markdown -jaraco-classes==3.2.2 - # via keyring -jinja2==3.1.2 - # via - # cookiecutter - # feast - # jinja2-time - # pandas-profiling -jinja2-time==0.2.0 - # via cookiecutter -jmespath==0.10.0 - # via - # boto3 - # botocore -joblib==1.1.0 - # via - # flytekit - # pandas-profiling - # phik - # scikit-learn -jsonschema==4.16.0 - # via feast -keyring==23.9.3 - # via flytekit -kiwisolver==1.4.4 - # via matplotlib -locket==1.0.0 - # via partd -markdown==3.4.1 - # via flytekitplugins-deck-standard -markupsafe==2.1.1 - # via jinja2 -marshmallow==3.18.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -matplotlib==3.5.3 - # via - # -r ../../../common/requirements-common.in - # missingno - # pandas-profiling - # phik - # seaborn -missingno==0.5.1 - # via pandas-profiling -mmh3==3.0.0 - # via feast -more-itertools==8.14.0 - # via jaraco-classes -moreorless==0.4.0 - # via bowler -multidict==6.0.2 - # via - # aiohttp - # yarl -multimethod==1.8 - # via - # pandas-profiling - # visions -mypy==0.971 - # via sqlalchemy -mypy-extensions==0.4.3 - # via - # mypy - # typing-inspect -natsort==8.2.0 - # via flytekit -networkx==2.8.6 - # via visions -numpy==1.23.3 - # via - # -r requirements.in - # feast - # imagehash - # matplotlib - # missingno - # pandas - # pandas-profiling - # pandavro - # patsy - # phik - # pyarrow - # scikit-learn - # scipy - # seaborn - # statsmodels - # visions -packaging==21.3 - # via - # dask - # docker - # marshmallow - # matplotlib - # statsmodels -pandas==1.4.4 - # via - # feast - # flytekit - # pandas-profiling - # pandavro - # phik - # seaborn - # statsmodels - # visions -pandas-profiling==3.3.0 - # via flytekitplugins-deck-standard -pandavro==1.5.2 - # via feast -partd==1.3.0 - # via dask -patsy==0.5.2 - # via statsmodels -phik==0.12.2 - # via pandas-profiling -pillow==9.2.0 - # via - # imagehash - # matplotlib - # visions -plotly==5.10.0 - # via flytekitplugins-deck-standard -proto-plus==1.22.1 - # via feast -protobuf==3.20.2 - # via - # feast - # flyteidl - # flytekit - # google-api-core - # googleapis-common-protos - # grpcio-reflection - # grpcio-status - # proto-plus - # protoc-gen-swagger - # tensorflow-metadata -protoc-gen-swagger==0.1.0 - # via flyteidl -py==1.11.0 - # via retry -pyarrow==6.0.1 - # via - # feast - # flytekit -pyasn1==0.4.8 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.2.8 - # via google-auth -pycparser==2.21 - # via cffi -pydantic==1.9.2 - # via - # fastapi - # feast - # pandas-profiling -pygments==2.13.0 - # via feast -pyopenssl==22.0.0 - # via flytekit -pyparsing==3.0.9 - # via - # matplotlib - # packaging -pyrsistent==0.18.1 - # via jsonschema -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # flytekit - # matplotlib - # pandas -python-dotenv==0.21.0 - # via uvicorn -python-json-logger==2.0.4 - # via flytekit -python-slugify==6.1.2 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.2.1 - # via - # flytekit - # pandas -pywavelets==1.4.1 - # via imagehash -pyyaml==6.0 - # via - # cookiecutter - # dask - # feast - # flytekit - # pandas-profiling - # uvicorn -regex==2022.9.13 - # via docker-image-py -requests==2.28.1 - # via - # cookiecutter - # docker - # flytekit - # google-api-core - # pandas-profiling - # responses -responses==0.21.0 - # via flytekit -retry==0.9.2 - # via flytekit -rsa==4.9 - # via google-auth -s3fs==2022.1.0 - # via feast -s3transfer==0.5.2 - # via boto3 -scikit-learn==1.1.2 - # via -r requirements.in -scipy==1.9.1 - # via - # imagehash - # missingno - # pandas-profiling - # phik - # scikit-learn - # seaborn - # statsmodels -seaborn==0.11.2 - # via - # missingno - # pandas-profiling -six==1.16.0 - # via - # google-auth - # grpcio - # pandavro - # patsy - # python-dateutil -sniffio==1.3.0 - # via anyio -sortedcontainers==2.4.0 - # via flytekit -sqlalchemy[mypy]==1.4.41 - # via feast -sqlalchemy2-stubs==0.0.2a27 - # via sqlalchemy -starlette==0.20.4 - # via fastapi -statsd==3.3.0 - # via flytekit -statsmodels==0.13.2 - # via pandas-profiling -tabulate==0.8.10 - # via feast -tangled-up-in-unicode==0.2.0 - # via - # pandas-profiling - # visions -tenacity==8.1.0 - # via - # feast - # plotly -tensorflow-metadata==1.10.0 - # via feast -text-unidecode==1.3 - # via python-slugify -threadpoolctl==3.1.0 - # via scikit-learn -toml==0.10.2 - # via feast -tomli==2.0.1 - # via mypy -toolz==0.12.0 - # via - # dask - # partd -tqdm==4.64.1 - # via - # feast - # pandas-profiling -typeguard==2.13.3 - # via feast -typing-extensions==4.3.0 - # via - # aioitertools - # flytekit - # mypy - # pydantic - # sqlalchemy2-stubs - # starlette - # typing-inspect -typing-inspect==0.8.0 - # via dataclasses-json -urllib3==1.26.12 - # via - # botocore - # docker - # flytekit - # requests - # responses -uvicorn[standard]==0.18.3 - # via feast -uvloop==0.17.0 - # via uvicorn -visions[type_image_path]==0.7.5 - # via pandas-profiling -volatile==2.1.0 - # via bowler -watchfiles==0.17.0 - # via uvicorn -websocket-client==1.4.1 - # via docker -websockets==10.3 - # via uvicorn -wheel==0.37.1 - # via - # -r ../../../common/requirements-common.in - # flytekit -wrapt==1.14.1 - # via - # aiobotocore - # deprecated - # flytekit -yarl==1.8.1 - # via aiohttp -zipp==3.8.1 - # via importlib-metadata diff --git a/examples/hive_plugin/requirements.in b/examples/hive_plugin/requirements.in new file mode 100644 index 000000000..25b37bfdb --- /dev/null +++ b/examples/hive_plugin/requirements.in @@ -0,0 +1 @@ +flytekitplugins-hive diff --git a/examples/k8s_spark_plugin/requirements.txt b/examples/k8s_spark_plugin/requirements.in similarity index 58% rename from examples/k8s_spark_plugin/requirements.txt rename to examples/k8s_spark_plugin/requirements.in index 298112464..e83418e00 100644 --- a/examples/k8s_spark_plugin/requirements.txt +++ b/examples/k8s_spark_plugin/requirements.in @@ -1,3 +1,2 @@ flytekitplugins-spark -flytekitplugins-envd pandas diff --git a/examples/kfmpi_plugin/requirements.in b/examples/kfmpi_plugin/requirements.in index a41edca38..4bd4c68a5 100644 --- a/examples/kfmpi_plugin/requirements.in +++ b/examples/kfmpi_plugin/requirements.in @@ -1,2 +1,2 @@ flytekitplugins-kfmpi -flytekitplugins-envd +tensorflow diff --git a/examples/kfmpi_plugin/requirements.txt b/examples/kfmpi_plugin/requirements.txt deleted file mode 100644 index 6c9aa3a64..000000000 --- a/examples/kfmpi_plugin/requirements.txt +++ /dev/null @@ -1,355 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile requirements.in -# -adlfs==2023.8.0 - # via flytekit -aiobotocore==2.5.4 - # via s3fs -aiohttp==3.9.2 - # via - # adlfs - # aiobotocore - # gcsfs - # s3fs -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp -arrow==1.2.3 - # via cookiecutter -async-timeout==4.0.3 - # via aiohttp -attrs==23.1.0 - # via aiohttp -azure-core==1.29.2 - # via - # adlfs - # azure-identity - # azure-storage-blob -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.14.0 - # via adlfs -azure-storage-blob==12.17.0 - # via adlfs -binaryornot==0.4.4 - # via cookiecutter -botocore==1.31.17 - # via aiobotocore -cachetools==5.3.1 - # via google-auth -certifi==2023.7.22 - # via - # kubernetes - # requests -cffi==1.15.1 - # via - # azure-datalake-store - # cryptography -chardet==5.2.0 - # via binaryornot -charset-normalizer==3.2.0 - # via requests -click==8.1.6 - # via - # cookiecutter - # flytekit - # rich-click -cloudpickle==2.2.1 - # via flytekit -cookiecutter==2.3.0 - # via flytekit -croniter==1.4.1 - # via flytekit -cryptography==41.0.3 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt - # pyopenssl - # secretstorage -dataclasses-json==0.5.9 - # via flytekit -decorator==5.1.1 - # via gcsfs -deprecated==1.2.14 - # via flytekit -diskcache==5.6.1 - # via flytekit -docker==6.1.3 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.15 - # via flytekit -envd==0.3.38 - # via flytekitplugins-envd -flyteidl==1.5.16 - # via flytekit -flytekit==1.9.0 - # via - # flytekitplugins-envd - # flytekitplugins-kfmpi -flytekitplugins-envd==1.9.0 - # via -r requirements.in -flytekitplugins-kfmpi==1.9.0 - # via -r requirements.in -frozenlist==1.4.0 - # via - # aiohttp - # aiosignal -fsspec==2023.6.0 - # via - # adlfs - # flytekit - # gcsfs - # s3fs -gcsfs==2023.6.0 - # via flytekit -gitdb==4.0.10 - # via gitpython -gitpython==3.1.32 - # via flytekit -google-api-core==2.11.1 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.22.0 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage - # kubernetes -google-auth-oauthlib==1.0.0 - # via gcsfs -google-cloud-core==2.3.3 - # via google-cloud-storage -google-cloud-storage==2.10.0 - # via gcsfs -google-crc32c==1.5.0 - # via google-resumable-media -google-resumable-media==2.5.0 - # via google-cloud-storage -googleapis-common-protos==1.60.0 - # via - # flyteidl - # flytekit - # google-api-core - # grpcio-status -grpcio==1.53.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.53.0 - # via flytekit -idna==3.4 - # via - # requests - # yarl -importlib-metadata==6.8.0 - # via - # flytekit - # keyring -isodate==0.6.1 - # via azure-storage-blob -jaraco-classes==3.3.0 - # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.2 - # via cookiecutter -jmespath==1.0.1 - # via botocore -joblib==1.3.2 - # via flytekit -keyring==24.2.0 - # via flytekit -kubernetes==27.2.0 - # via flytekit -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.3 - # via jinja2 -marshmallow==3.20.1 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via - # dataclasses-json - # flytekit -marshmallow-jsonschema==0.13.0 - # via flytekit -mdurl==0.1.2 - # via markdown-it-py -more-itertools==10.1.0 - # via jaraco-classes -msal==1.23.0 - # via - # azure-datalake-store - # azure-identity - # msal-extensions -msal-extensions==1.0.0 - # via azure-identity -multidict==6.0.4 - # via - # aiohttp - # yarl -mypy-extensions==1.0.0 - # via typing-inspect -natsort==8.4.0 - # via flytekit -numpy==1.25.2 - # via - # flytekit - # pandas - # pyarrow -oauthlib==3.2.2 - # via - # kubernetes - # requests-oauthlib -packaging==23.1 - # via - # docker - # marshmallow -pandas==1.5.3 - # via flytekit -portalocker==2.7.0 - # via msal-extensions -protobuf==4.24.0 - # via - # flyteidl - # google-api-core - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -pyarrow==10.0.1 - # via flytekit -pyasn1==0.5.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pycparser==2.21 - # via cffi -pygments==2.16.1 - # via rich -pyjwt[crypto]==2.8.0 - # via - # msal - # pyjwt -pyopenssl==23.2.0 - # via flytekit -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # flytekit - # kubernetes - # pandas -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2023.3 - # via - # flytekit - # pandas -pyyaml==6.0.1 - # via - # cookiecutter - # flytekit - # kubernetes -regex==2023.8.8 - # via docker-image-py -requests==2.31.0 - # via - # azure-core - # azure-datalake-store - # cookiecutter - # docker - # flytekit - # gcsfs - # google-api-core - # google-cloud-storage - # kubernetes - # msal - # requests-oauthlib -requests-oauthlib==1.3.1 - # via - # google-auth-oauthlib - # kubernetes -rich==13.5.2 - # via - # cookiecutter - # flytekit - # rich-click -rich-click==1.6.1 - # via flytekit -rsa==4.9 - # via google-auth -s3fs==2023.6.0 - # via flytekit -secretstorage==3.3.3 - # via keyring -six==1.16.0 - # via - # azure-core - # google-auth - # isodate - # kubernetes - # python-dateutil -smmap==5.0.0 - # via gitdb -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -typing-extensions==4.7.1 - # via - # azure-core - # azure-storage-blob - # flytekit - # typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -urllib3==1.26.16 - # via - # botocore - # docker - # flytekit - # google-auth - # kubernetes - # requests -websocket-client==1.6.1 - # via - # docker - # kubernetes -wheel==0.41.1 - # via flytekit -wrapt==1.15.0 - # via - # aiobotocore - # deprecated - # flytekit -yarl==1.9.2 - # via aiohttp -zipp==3.16.2 - # via importlib-metadata diff --git a/examples/kfpytorch_plugin/requirements.in b/examples/kfpytorch_plugin/requirements.in index a55846894..9599531ea 100644 --- a/examples/kfpytorch_plugin/requirements.in +++ b/examples/kfpytorch_plugin/requirements.in @@ -1,4 +1,4 @@ flytekit==1.10.3 flytekitplugins-kfpytorch -flytekitplugins-envd matplotlib +torch \ No newline at end of file diff --git a/examples/kfpytorch_plugin/requirements.txt b/examples/kfpytorch_plugin/requirements.txt deleted file mode 100644 index 9f64e2473..000000000 --- a/examples/kfpytorch_plugin/requirements.txt +++ /dev/null @@ -1,349 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile requirements.in -# -adlfs==2023.9.0 - # via flytekit -aiobotocore==2.5.4 - # via s3fs -aiohttp==3.9.3 - # via - # adlfs - # aiobotocore - # gcsfs - # s3fs -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp -arrow==1.3.0 - # via cookiecutter -async-timeout==4.0.3 - # via aiohttp -attrs==23.2.0 - # via aiohttp -azure-core==1.30.0 - # via - # adlfs - # azure-identity - # azure-storage-blob -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.15.0 - # via adlfs -azure-storage-blob==12.19.0 - # via adlfs -binaryornot==0.4.4 - # via cookiecutter -botocore==1.31.17 - # via aiobotocore -cachetools==5.3.2 - # via google-auth -certifi==2024.2.2 - # via - # kubernetes - # requests -cffi==1.16.0 - # via - # azure-datalake-store - # cryptography -chardet==5.2.0 - # via binaryornot -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # cookiecutter - # flytekit - # rich-click -cloudpickle==3.0.0 - # via - # flytekit - # flytekitplugins-kfpytorch -contourpy==1.2.0 - # via matplotlib -cookiecutter==2.5.0 - # via flytekit -croniter==2.0.1 - # via flytekit -cryptography==42.0.2 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt -cycler==0.12.1 - # via matplotlib -dataclasses-json==0.5.9 - # via flytekit -decorator==5.1.1 - # via gcsfs -diskcache==5.6.3 - # via flytekit -docker==6.1.3 - # via flytekit -docstring-parser==0.15 - # via flytekit -envd==0.3.45 - # via flytekitplugins-envd -flyteidl==1.10.6 - # via - # flytekit - # flytekitplugins-kfpytorch -flytekit==1.10.3 - # via - # -r requirements.in - # flytekitplugins-envd - # flytekitplugins-kfpytorch -flytekitplugins-envd==1.10.3 - # via -r requirements.in -flytekitplugins-kfpytorch==1.10.3 - # via -r requirements.in -fonttools==4.48.1 - # via matplotlib -frozenlist==1.4.1 - # via - # aiohttp - # aiosignal -fsspec==2023.9.2 - # via - # adlfs - # flytekit - # gcsfs - # s3fs -gcsfs==2023.9.2 - # via flytekit -google-api-core==2.16.2 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.27.0 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage - # kubernetes -google-auth-oauthlib==1.2.0 - # via gcsfs -google-cloud-core==2.4.1 - # via google-cloud-storage -google-cloud-storage==2.14.0 - # via gcsfs -google-crc32c==1.5.0 - # via - # google-cloud-storage - # google-resumable-media -google-resumable-media==2.7.0 - # via google-cloud-storage -googleapis-common-protos==1.62.0 - # via - # flyteidl - # flytekit - # google-api-core - # grpcio-status -grpcio==1.60.1 - # via - # flytekit - # grpcio-status -grpcio-status==1.60.1 - # via flytekit -idna==3.6 - # via - # requests - # yarl -importlib-metadata==7.0.1 - # via - # flytekit - # keyring -isodate==0.6.1 - # via azure-storage-blob -jaraco-classes==3.3.1 - # via keyring -jinja2==3.1.3 - # via cookiecutter -jmespath==1.0.1 - # via botocore -joblib==1.3.2 - # via flytekit -jsonpickle==3.0.2 - # via flytekit -keyring==24.3.0 - # via flytekit -kiwisolver==1.4.5 - # via matplotlib -kubernetes==29.0.0 - # via flytekit -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.5 - # via jinja2 -marshmallow==3.20.2 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via - # dataclasses-json - # flytekit -marshmallow-jsonschema==0.13.0 - # via flytekit -mashumaro==3.12 - # via flytekit -matplotlib==3.8.2 - # via -r requirements.in -mdurl==0.1.2 - # via markdown-it-py -more-itertools==10.2.0 - # via jaraco-classes -msal==1.26.0 - # via - # azure-datalake-store - # azure-identity - # msal-extensions -msal-extensions==1.1.0 - # via azure-identity -multidict==6.0.5 - # via - # aiohttp - # yarl -mypy-extensions==1.0.0 - # via typing-inspect -numpy==1.26.4 - # via - # contourpy - # matplotlib - # pyarrow -oauthlib==3.2.2 - # via - # kubernetes - # requests-oauthlib -packaging==23.2 - # via - # docker - # marshmallow - # matplotlib - # msal-extensions -pillow==10.2.0 - # via matplotlib -portalocker==2.8.2 - # via msal-extensions -protobuf==4.24.4 - # via - # flyteidl - # flytekit - # google-api-core - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -pyarrow==15.0.0 - # via flytekit -pyasn1==0.5.1 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pycparser==2.21 - # via cffi -pygments==2.17.2 - # via rich -pyjwt[crypto]==2.8.0 - # via - # msal - # pyjwt -pyparsing==3.1.1 - # via matplotlib -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # kubernetes - # matplotlib -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.3 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2024.1 - # via croniter -pyyaml==6.0.1 - # via - # cookiecutter - # flytekit - # kubernetes -requests==2.31.0 - # via - # azure-core - # azure-datalake-store - # cookiecutter - # docker - # flytekit - # gcsfs - # google-api-core - # google-cloud-storage - # kubernetes - # msal - # requests-oauthlib -requests-oauthlib==1.3.1 - # via - # google-auth-oauthlib - # kubernetes -rich==13.7.0 - # via - # cookiecutter - # flytekit - # rich-click -rich-click==1.7.3 - # via flytekit -rsa==4.9 - # via google-auth -s3fs==2023.9.2 - # via flytekit -six==1.16.0 - # via - # azure-core - # isodate - # kubernetes - # python-dateutil -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -types-python-dateutil==2.8.19.20240106 - # via arrow -typing-extensions==4.9.0 - # via - # azure-core - # azure-storage-blob - # flytekit - # mashumaro - # rich-click - # typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -urllib3==1.26.18 - # via - # botocore - # docker - # flytekit - # kubernetes - # requests -websocket-client==1.7.0 - # via - # docker - # kubernetes -wrapt==1.16.0 - # via aiobotocore -yarl==1.9.4 - # via aiohttp -zipp==3.17.0 - # via importlib-metadata diff --git a/examples/kftensorflow_plugin/requirements.in b/examples/kftensorflow_plugin/requirements.in index c03bf9dee..2bcb2657f 100644 --- a/examples/kftensorflow_plugin/requirements.in +++ b/examples/kftensorflow_plugin/requirements.in @@ -1,3 +1,3 @@ tensorflow-datasets flytekitplugins-kftensorflow -flytekitplugins-envd +tensorflow diff --git a/examples/kftensorflow_plugin/requirements.txt b/examples/kftensorflow_plugin/requirements.txt deleted file mode 100644 index 2437ccb70..000000000 --- a/examples/kftensorflow_plugin/requirements.txt +++ /dev/null @@ -1,396 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile requirements.in -# -absl-py==1.4.0 - # via - # array-record - # tensorflow-datasets - # tensorflow-metadata -adlfs==2023.8.0 - # via flytekit -aiobotocore==2.5.4 - # via s3fs -aiohttp==3.8.5 - # via - # adlfs - # aiobotocore - # gcsfs - # s3fs -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp -array-record==0.4.1 - # via tensorflow-datasets -arrow==1.2.3 - # via cookiecutter -async-timeout==4.0.3 - # via aiohttp -attrs==23.1.0 - # via aiohttp -azure-core==1.29.2 - # via - # adlfs - # azure-identity - # azure-storage-blob -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.14.0 - # via adlfs -azure-storage-blob==12.17.0 - # via adlfs -binaryornot==0.4.4 - # via cookiecutter -botocore==1.31.17 - # via aiobotocore -cachetools==5.3.1 - # via google-auth -certifi==2023.7.22 - # via - # kubernetes - # requests -cffi==1.15.1 - # via - # azure-datalake-store - # cryptography -chardet==5.2.0 - # via binaryornot -charset-normalizer==3.2.0 - # via - # aiohttp - # requests -click==8.1.6 - # via - # cookiecutter - # flytekit - # rich-click - # tensorflow-datasets -cloudpickle==2.2.1 - # via flytekit -cookiecutter==2.3.0 - # via flytekit -croniter==1.4.1 - # via flytekit -cryptography==41.0.3 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt - # pyopenssl - # secretstorage -dataclasses-json==0.5.9 - # via flytekit -decorator==5.1.1 - # via gcsfs -deprecated==1.2.14 - # via flytekit -diskcache==5.6.1 - # via flytekit -dm-tree==0.1.8 - # via tensorflow-datasets -docker==6.1.3 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.15 - # via flytekit -envd==0.3.38 - # via flytekitplugins-envd -etils[enp,epath,epy]==1.4.1 - # via - # array-record - # tensorflow-datasets -flyteidl==1.5.16 - # via flytekit -flytekit==1.9.0 - # via - # flytekitplugins-envd - # flytekitplugins-kftensorflow -flytekitplugins-envd==1.9.0 - # via -r requirements.in -flytekitplugins-kftensorflow==1.9.0 - # via -r requirements.in -frozenlist==1.4.0 - # via - # aiohttp - # aiosignal -fsspec==2023.6.0 - # via - # adlfs - # flytekit - # gcsfs - # s3fs -gcsfs==2023.6.0 - # via flytekit -gitdb==4.0.10 - # via gitpython -gitpython==3.1.37 - # via flytekit -google-api-core==2.11.1 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.22.0 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage - # kubernetes -google-auth-oauthlib==1.0.0 - # via gcsfs -google-cloud-core==2.3.3 - # via google-cloud-storage -google-cloud-storage==2.10.0 - # via gcsfs -google-crc32c==1.5.0 - # via google-resumable-media -google-resumable-media==2.5.0 - # via google-cloud-storage -googleapis-common-protos==1.60.0 - # via - # flyteidl - # flytekit - # google-api-core - # grpcio-status - # tensorflow-metadata -grpcio==1.53.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.53.0 - # via flytekit -idna==3.4 - # via - # requests - # yarl -importlib-metadata==6.8.0 - # via - # flytekit - # keyring -importlib-resources==6.0.1 - # via etils -isodate==0.6.1 - # via azure-storage-blob -jaraco-classes==3.3.0 - # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.2 - # via cookiecutter -jmespath==1.0.1 - # via botocore -joblib==1.3.2 - # via flytekit -keyring==24.2.0 - # via flytekit -kubernetes==27.2.0 - # via flytekit -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.3 - # via jinja2 -marshmallow==3.20.1 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via - # dataclasses-json - # flytekit -marshmallow-jsonschema==0.13.0 - # via flytekit -mdurl==0.1.2 - # via markdown-it-py -more-itertools==10.1.0 - # via jaraco-classes -msal==1.23.0 - # via - # azure-datalake-store - # azure-identity - # msal-extensions -msal-extensions==1.0.0 - # via azure-identity -multidict==6.0.4 - # via - # aiohttp - # yarl -mypy-extensions==1.0.0 - # via typing-inspect -natsort==8.4.0 - # via flytekit -numpy==1.25.2 - # via - # etils - # flytekit - # pandas - # pyarrow - # tensorflow-datasets -oauthlib==3.2.2 - # via - # kubernetes - # requests-oauthlib -packaging==23.1 - # via - # docker - # marshmallow -pandas==1.5.3 - # via flytekit -portalocker==2.7.0 - # via msal-extensions -promise==2.3 - # via tensorflow-datasets -protobuf==4.24.0 - # via - # flyteidl - # google-api-core - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger - # tensorflow-datasets - # tensorflow-metadata -protoc-gen-swagger==0.1.0 - # via flyteidl -psutil==5.9.5 - # via tensorflow-datasets -pyarrow==10.0.1 - # via flytekit -pyasn1==0.5.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pycparser==2.21 - # via cffi -pygments==2.16.1 - # via rich -pyjwt[crypto]==2.8.0 - # via msal -pyopenssl==23.2.0 - # via flytekit -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # flytekit - # kubernetes - # pandas -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2023.3 - # via - # flytekit - # pandas -pyyaml==6.0.1 - # via - # cookiecutter - # flytekit - # kubernetes -regex==2023.8.8 - # via docker-image-py -requests==2.31.0 - # via - # azure-core - # azure-datalake-store - # cookiecutter - # docker - # flytekit - # gcsfs - # google-api-core - # google-cloud-storage - # kubernetes - # msal - # requests-oauthlib - # tensorflow-datasets -requests-oauthlib==1.3.1 - # via - # google-auth-oauthlib - # kubernetes -rich==13.5.2 - # via - # cookiecutter - # flytekit - # rich-click -rich-click==1.6.1 - # via flytekit -rsa==4.9 - # via google-auth -s3fs==2023.6.0 - # via flytekit -secretstorage==3.3.3 - # via keyring -six==1.16.0 - # via - # azure-core - # google-auth - # isodate - # kubernetes - # promise - # python-dateutil -smmap==5.0.0 - # via gitdb -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -tensorflow-datasets==4.9.2 - # via -r requirements.in -tensorflow-metadata==1.13.1 - # via tensorflow-datasets -termcolor==2.3.0 - # via tensorflow-datasets -text-unidecode==1.3 - # via python-slugify -toml==0.10.2 - # via tensorflow-datasets -tqdm==4.66.1 - # via tensorflow-datasets -typing-extensions==4.7.1 - # via - # azure-core - # azure-storage-blob - # etils - # flytekit - # typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -urllib3==1.26.16 - # via - # botocore - # docker - # flytekit - # google-auth - # kubernetes - # requests -websocket-client==1.6.1 - # via - # docker - # kubernetes -wheel==0.41.1 - # via flytekit -wrapt==1.15.0 - # via - # aiobotocore - # deprecated - # flytekit - # tensorflow-datasets -yarl==1.9.2 - # via aiohttp -zipp==3.16.2 - # via - # etils - # importlib-metadata diff --git a/examples/mnist_classifier/mnist_classifier/pytorch_single_node_and_gpu.py b/examples/mnist_classifier/mnist_classifier/pytorch_single_node_and_gpu.py index 0d9382fdb..997e5122b 100644 --- a/examples/mnist_classifier/mnist_classifier/pytorch_single_node_and_gpu.py +++ b/examples/mnist_classifier/mnist_classifier/pytorch_single_node_and_gpu.py @@ -267,24 +267,20 @@ class Hyperparameters(object): # %% if os.getenv("SANDBOX") != "": print(f"SANDBOX ENV: '{os.getenv('SANDBOX')}'") - mem = "100Mi" gpu = "0" - storage = "500Mi" else: print(f"SANDBOX ENV: '{os.getenv('SANDBOX')}'") - mem = "3Gi" gpu = "1" - storage = "1Gi" @task( retries=2, cache=True, cache_version="1.0", - requests=Resources(gpu=gpu, mem=mem, storage=storage), - limits=Resources(gpu=gpu, mem=mem, storage=storage), + requests=Resources(gpu=gpu, mem=mem), + limits=Resources(gpu=gpu, mem=mem), ) def pytorch_mnist_task(hp: Hyperparameters) -> TrainingOutputs: wandb_setup() diff --git a/examples/modin_plugin/requirements.in b/examples/modin_plugin/requirements.in index c109e76aa..98427587c 100644 --- a/examples/modin_plugin/requirements.in +++ b/examples/modin_plugin/requirements.in @@ -4,3 +4,4 @@ matplotlib flytekitplugins-deck-standard flytekitplugins-modin scikit-learn +modin diff --git a/examples/sagemaker_inference_agent/requirements.in b/examples/sagemaker_inference_agent/requirements.in index c2b650401..ed114b348 100644 --- a/examples/sagemaker_inference_agent/requirements.in +++ b/examples/sagemaker_inference_agent/requirements.in @@ -3,3 +3,5 @@ xgboost fastapi uvicorn scikit-learn +flytekit==1.11.0 +flyteidl==1.11.1b0 diff --git a/examples/snowflake_agent/requirements.in b/examples/snowflake_agent/requirements.in index 20d2f99d4..0353f65ed 100644 --- a/examples/snowflake_agent/requirements.in +++ b/examples/snowflake_agent/requirements.in @@ -1,2 +1,2 @@ flytekitplugins-snowflake==1.7.0 -flytekit==1.7.0 +flytekit==1.7.1b1 diff --git a/examples/snowflake_plugin/requirements.in b/examples/snowflake_plugin/requirements.in index 20d2f99d4..0353f65ed 100644 --- a/examples/snowflake_plugin/requirements.in +++ b/examples/snowflake_plugin/requirements.in @@ -1,2 +1,2 @@ flytekitplugins-snowflake==1.7.0 -flytekit==1.7.0 +flytekit==1.7.1b1 From 3162b1bdaa28beb1abfa615b76258919c3a1562b Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 17:32:38 +0800 Subject: [PATCH 26/47] test Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 7 +- examples/bigquery_agent/requirements.in | 2 +- examples/bigquery_agent/requirements.txt | 246 ------------------ examples/bigquery_plugin/requirements.in | 2 +- examples/bigquery_plugin/requirements.txt | 246 ------------------ .../customizing_dependencies/requirements.in | 3 +- 6 files changed, 9 insertions(+), 497 deletions(-) delete mode 100644 examples/bigquery_agent/requirements.txt delete mode 100644 examples/bigquery_plugin/requirements.txt diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 80ec7bffb..e89cafc9f 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -76,4 +76,9 @@ jobs: run: | pip freeze source .venv/bin/activate - pyflyte -vvv --pkgs ${{ matrix.example }} package -o spark-package.tgz --force --fast + pyflyte -vvv \ + --pkgs ${{ matrix.example }} package \ + --image mindmeld="ghcr.io/flyteorg/flytecookbook:core-latest" \ + --image borebuster="ghcr.io/flyteorg/flytekit:py3.9-latest" \ + --output spark-package.tgz \ + --force --fast diff --git a/examples/bigquery_agent/requirements.in b/examples/bigquery_agent/requirements.in index 0b590068f..a987746f1 100644 --- a/examples/bigquery_agent/requirements.in +++ b/examples/bigquery_agent/requirements.in @@ -2,4 +2,4 @@ flytekit wheel matplotlib flytekitplugins-deck-standard -flytekitplugins-bigquery==0.30.1 +flytekitplugins-bigquery diff --git a/examples/bigquery_agent/requirements.txt b/examples/bigquery_agent/requirements.txt deleted file mode 100644 index e8556d240..000000000 --- a/examples/bigquery_agent/requirements.txt +++ /dev/null @@ -1,246 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# /Library/Developer/CommandLineTools/usr/bin/make requirements.txt -# -arrow==1.2.2 - # via jinja2-time -binaryornot==0.4.4 - # via cookiecutter -cachetools==5.0.0 - # via google-auth -certifi==2021.10.8 - # via requests -cffi==1.15.0 - # via cryptography -chardet==4.0.0 - # via binaryornot -charset-normalizer==2.0.12 - # via requests -checksumdir==1.2.0 - # via flytekit -click==8.1.2 - # via - # cookiecutter - # flytekit -cloudpickle==2.0.0 - # via flytekit -cookiecutter==2.1.1 - # via flytekit -croniter==1.3.4 - # via flytekit -cryptography==37.0.3 - # via secretstorage -cycler==0.11.0 - # via matplotlib -dataclasses-json==0.5.7 - # via flytekit -decorator==5.1.1 - # via retry -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.13 - # via flytekit -flyteidl==0.24.21 - # via flytekit -flytekit==0.32.3 - # via - # -r ../../../common/requirements-common.in - # flytekitplugins-bigquery -flytekitplugins-bigquery==0.30.1 - # via -r requirements.in -fonttools==4.32.0 - # via matplotlib -google-api-core[grpc]==2.7.1 - # via - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-core -google-auth==2.6.4 - # via - # google-api-core - # google-cloud-core -google-cloud-bigquery==3.0.1 - # via flytekitplugins-bigquery -google-cloud-bigquery-storage==2.13.1 - # via google-cloud-bigquery -google-cloud-core==2.3.0 - # via google-cloud-bigquery -google-crc32c==1.3.0 - # via google-resumable-media -google-resumable-media==2.3.2 - # via google-cloud-bigquery -googleapis-common-protos==1.56.0 - # via - # flyteidl - # google-api-core - # grpcio-status -grpcio==1.44.0 - # via - # flytekit - # google-api-core - # google-cloud-bigquery - # grpcio-status -grpcio-status==1.44.0 - # via - # flytekit - # google-api-core -idna==3.3 - # via requests -importlib-metadata==4.11.3 - # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.1 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 - # via cookiecutter -keyring==23.5.0 - # via flytekit -kiwisolver==1.4.2 - # via matplotlib -markupsafe==2.1.1 - # via jinja2 -marshmallow==3.15.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -matplotlib==3.5.1 - # via -r ../../../common/requirements-common.in -mypy-extensions==0.4.3 - # via typing-inspect -natsort==8.1.0 - # via flytekit -numpy==1.22.3 - # via - # matplotlib - # pandas - # pyarrow -packaging==21.3 - # via - # google-cloud-bigquery - # marshmallow - # matplotlib -pandas==1.4.2 - # via flytekit -pillow==9.1.0 - # via matplotlib -proto-plus==1.20.3 - # via - # google-cloud-bigquery - # google-cloud-bigquery-storage -protobuf==3.20.0 - # via - # flyteidl - # flytekit - # google-api-core - # google-cloud-bigquery - # googleapis-common-protos - # grpcio-status - # proto-plus - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -py==1.11.0 - # via retry -pyarrow==6.0.1 - # via - # flytekit - # google-cloud-bigquery -pyasn1==0.4.8 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.2.8 - # via google-auth -pycparser==2.21 - # via cffi -pyparsing==3.0.8 - # via - # matplotlib - # packaging -python-dateutil==2.8.2 - # via - # arrow - # croniter - # flytekit - # google-cloud-bigquery - # matplotlib - # pandas -python-json-logger==2.0.2 - # via flytekit -python-slugify==6.1.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.1 - # via - # flytekit - # pandas -pyyaml==6.0 - # via - # cookiecutter - # flytekit -regex==2022.3.15 - # via docker-image-py -requests==2.27.1 - # via - # cookiecutter - # flytekit - # google-api-core - # google-cloud-bigquery - # responses -responses==0.20.0 - # via flytekit -retry==0.9.2 - # via flytekit -rsa==4.8 - # via google-auth -secretstorage==3.3.2 - # via keyring -six==1.16.0 - # via - # google-auth - # grpcio - # python-dateutil -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -typing-extensions==4.1.1 - # via - # flytekit - # typing-inspect -typing-inspect==0.7.1 - # via dataclasses-json -urllib3==1.26.9 - # via - # flytekit - # requests - # responses -wheel==0.37.1 - # via - # -r ../../../common/requirements-common.in - # flytekit -wrapt==1.14.0 - # via - # deprecated - # flytekit -zipp==3.8.0 - # via importlib-metadata diff --git a/examples/bigquery_plugin/requirements.in b/examples/bigquery_plugin/requirements.in index 0b590068f..a987746f1 100644 --- a/examples/bigquery_plugin/requirements.in +++ b/examples/bigquery_plugin/requirements.in @@ -2,4 +2,4 @@ flytekit wheel matplotlib flytekitplugins-deck-standard -flytekitplugins-bigquery==0.30.1 +flytekitplugins-bigquery diff --git a/examples/bigquery_plugin/requirements.txt b/examples/bigquery_plugin/requirements.txt deleted file mode 100644 index e8556d240..000000000 --- a/examples/bigquery_plugin/requirements.txt +++ /dev/null @@ -1,246 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# /Library/Developer/CommandLineTools/usr/bin/make requirements.txt -# -arrow==1.2.2 - # via jinja2-time -binaryornot==0.4.4 - # via cookiecutter -cachetools==5.0.0 - # via google-auth -certifi==2021.10.8 - # via requests -cffi==1.15.0 - # via cryptography -chardet==4.0.0 - # via binaryornot -charset-normalizer==2.0.12 - # via requests -checksumdir==1.2.0 - # via flytekit -click==8.1.2 - # via - # cookiecutter - # flytekit -cloudpickle==2.0.0 - # via flytekit -cookiecutter==2.1.1 - # via flytekit -croniter==1.3.4 - # via flytekit -cryptography==37.0.3 - # via secretstorage -cycler==0.11.0 - # via matplotlib -dataclasses-json==0.5.7 - # via flytekit -decorator==5.1.1 - # via retry -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.13 - # via flytekit -flyteidl==0.24.21 - # via flytekit -flytekit==0.32.3 - # via - # -r ../../../common/requirements-common.in - # flytekitplugins-bigquery -flytekitplugins-bigquery==0.30.1 - # via -r requirements.in -fonttools==4.32.0 - # via matplotlib -google-api-core[grpc]==2.7.1 - # via - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-core -google-auth==2.6.4 - # via - # google-api-core - # google-cloud-core -google-cloud-bigquery==3.0.1 - # via flytekitplugins-bigquery -google-cloud-bigquery-storage==2.13.1 - # via google-cloud-bigquery -google-cloud-core==2.3.0 - # via google-cloud-bigquery -google-crc32c==1.3.0 - # via google-resumable-media -google-resumable-media==2.3.2 - # via google-cloud-bigquery -googleapis-common-protos==1.56.0 - # via - # flyteidl - # google-api-core - # grpcio-status -grpcio==1.44.0 - # via - # flytekit - # google-api-core - # google-cloud-bigquery - # grpcio-status -grpcio-status==1.44.0 - # via - # flytekit - # google-api-core -idna==3.3 - # via requests -importlib-metadata==4.11.3 - # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.1 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 - # via cookiecutter -keyring==23.5.0 - # via flytekit -kiwisolver==1.4.2 - # via matplotlib -markupsafe==2.1.1 - # via jinja2 -marshmallow==3.15.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -matplotlib==3.5.1 - # via -r ../../../common/requirements-common.in -mypy-extensions==0.4.3 - # via typing-inspect -natsort==8.1.0 - # via flytekit -numpy==1.22.3 - # via - # matplotlib - # pandas - # pyarrow -packaging==21.3 - # via - # google-cloud-bigquery - # marshmallow - # matplotlib -pandas==1.4.2 - # via flytekit -pillow==9.1.0 - # via matplotlib -proto-plus==1.20.3 - # via - # google-cloud-bigquery - # google-cloud-bigquery-storage -protobuf==3.20.0 - # via - # flyteidl - # flytekit - # google-api-core - # google-cloud-bigquery - # googleapis-common-protos - # grpcio-status - # proto-plus - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -py==1.11.0 - # via retry -pyarrow==6.0.1 - # via - # flytekit - # google-cloud-bigquery -pyasn1==0.4.8 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.2.8 - # via google-auth -pycparser==2.21 - # via cffi -pyparsing==3.0.8 - # via - # matplotlib - # packaging -python-dateutil==2.8.2 - # via - # arrow - # croniter - # flytekit - # google-cloud-bigquery - # matplotlib - # pandas -python-json-logger==2.0.2 - # via flytekit -python-slugify==6.1.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.1 - # via - # flytekit - # pandas -pyyaml==6.0 - # via - # cookiecutter - # flytekit -regex==2022.3.15 - # via docker-image-py -requests==2.27.1 - # via - # cookiecutter - # flytekit - # google-api-core - # google-cloud-bigquery - # responses -responses==0.20.0 - # via flytekit -retry==0.9.2 - # via flytekit -rsa==4.8 - # via google-auth -secretstorage==3.3.2 - # via keyring -six==1.16.0 - # via - # google-auth - # grpcio - # python-dateutil -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -typing-extensions==4.1.1 - # via - # flytekit - # typing-inspect -typing-inspect==0.7.1 - # via dataclasses-json -urllib3==1.26.9 - # via - # flytekit - # requests - # responses -wheel==0.37.1 - # via - # -r ../../../common/requirements-common.in - # flytekit -wrapt==1.14.0 - # via - # deprecated - # flytekit -zipp==3.8.0 - # via importlib-metadata diff --git a/examples/customizing_dependencies/requirements.in b/examples/customizing_dependencies/requirements.in index 8705f3c93..6c0a722a5 100644 --- a/examples/customizing_dependencies/requirements.in +++ b/examples/customizing_dependencies/requirements.in @@ -1,3 +1,2 @@ -flytekit==1.9.0 -flytekitplugins-envd +flytekit scikit-learn From ce7b80778d49e9c827cd7d914934815817891b3c Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 17:45:03 +0800 Subject: [PATCH 27/47] test Signed-off-by: Kevin Su --- examples/customizing_dependencies/requirements.in | 1 + examples/data_types_and_io/requirements.in | 3 ++- examples/feast_integration/requirements.in | 2 +- examples/kfpytorch_plugin/requirements.in | 3 ++- .../mnist_classifier/pytorch_single_node_multi_gpu.py | 6 ++---- examples/modin_plugin/requirements.in | 1 + examples/sagemaker_inference_agent/requirements.in | 2 +- examples/snowflake_agent/requirements.in | 1 + examples/snowflake_plugin/requirements.in | 1 + 9 files changed, 12 insertions(+), 8 deletions(-) diff --git a/examples/customizing_dependencies/requirements.in b/examples/customizing_dependencies/requirements.in index 6c0a722a5..0ae7ad4e6 100644 --- a/examples/customizing_dependencies/requirements.in +++ b/examples/customizing_dependencies/requirements.in @@ -1,2 +1,3 @@ flytekit scikit-learn +pandas diff --git a/examples/data_types_and_io/requirements.in b/examples/data_types_and_io/requirements.in index 1411a4a0b..95e754537 100644 --- a/examples/data_types_and_io/requirements.in +++ b/examples/data_types_and_io/requirements.in @@ -1 +1,2 @@ -pandas \ No newline at end of file +pandas +torch diff --git a/examples/feast_integration/requirements.in b/examples/feast_integration/requirements.in index 436da3159..0c72f86b3 100644 --- a/examples/feast_integration/requirements.in +++ b/examples/feast_integration/requirements.in @@ -4,5 +4,5 @@ matplotlib flytekitplugins-deck-standard scikit-learn numpy -feast[aws] boto3 +feast[aws]==0.25.0 diff --git a/examples/kfpytorch_plugin/requirements.in b/examples/kfpytorch_plugin/requirements.in index 9599531ea..0d91657c0 100644 --- a/examples/kfpytorch_plugin/requirements.in +++ b/examples/kfpytorch_plugin/requirements.in @@ -1,4 +1,5 @@ flytekit==1.10.3 flytekitplugins-kfpytorch matplotlib -torch \ No newline at end of file +torch +tensorboardX diff --git a/examples/mnist_classifier/mnist_classifier/pytorch_single_node_multi_gpu.py b/examples/mnist_classifier/mnist_classifier/pytorch_single_node_multi_gpu.py index 4a1856643..63110fb18 100644 --- a/examples/mnist_classifier/mnist_classifier/pytorch_single_node_multi_gpu.py +++ b/examples/mnist_classifier/mnist_classifier/pytorch_single_node_multi_gpu.py @@ -369,21 +369,19 @@ def train_mnist(rank: int, world_size: int, hp: Hyperparameters): if os.getenv("SANDBOX") != "": mem = "100Mi" gpu = "0" - storage = "500Mi" ephemeral_storage = "500Mi" else: mem = "30Gi" gpu = str(WORLD_SIZE) ephemeral_storage = "500Mi" - storage = "20Gi" @task( retries=2, cache=True, cache_version="1.2", - requests=Resources(gpu=gpu, mem=mem, storage=storage, ephemeral_storage=ephemeral_storage), - limits=Resources(gpu=gpu, mem=mem, storage=storage, ephemeral_storage=ephemeral_storage), + requests=Resources(gpu=gpu, mem=mem, ephemeral_storage=ephemeral_storage), + limits=Resources(gpu=gpu, mem=mem, ephemeral_storage=ephemeral_storage), ) def pytorch_mnist_task(hp: Hyperparameters) -> TrainingOutputs: print("Start MNIST training:") diff --git a/examples/modin_plugin/requirements.in b/examples/modin_plugin/requirements.in index 98427587c..bc60bba02 100644 --- a/examples/modin_plugin/requirements.in +++ b/examples/modin_plugin/requirements.in @@ -5,3 +5,4 @@ flytekitplugins-deck-standard flytekitplugins-modin scikit-learn modin +ray diff --git a/examples/sagemaker_inference_agent/requirements.in b/examples/sagemaker_inference_agent/requirements.in index ed114b348..1160374e9 100644 --- a/examples/sagemaker_inference_agent/requirements.in +++ b/examples/sagemaker_inference_agent/requirements.in @@ -1,4 +1,4 @@ -flytekitplugins-awssagemaker>=1.12.0b2 +flytekitplugins-awssagemaker==1.12.0b2 xgboost fastapi uvicorn diff --git a/examples/snowflake_agent/requirements.in b/examples/snowflake_agent/requirements.in index 0353f65ed..4f50a7b41 100644 --- a/examples/snowflake_agent/requirements.in +++ b/examples/snowflake_agent/requirements.in @@ -1,2 +1,3 @@ flytekitplugins-snowflake==1.7.0 +flytekitplugins-envd==1.9.0 flytekit==1.7.1b1 diff --git a/examples/snowflake_plugin/requirements.in b/examples/snowflake_plugin/requirements.in index 0353f65ed..bbd72c042 100644 --- a/examples/snowflake_plugin/requirements.in +++ b/examples/snowflake_plugin/requirements.in @@ -1,2 +1,3 @@ flytekitplugins-snowflake==1.7.0 flytekit==1.7.1b1 +flytekitplugins-envd==1.9.0 From 6384e78412a94498d8bdd4e3aaa3b72069f4cfdb Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 17:53:26 +0800 Subject: [PATCH 28/47] test Signed-off-by: Kevin Su --- examples/kfpytorch_plugin/requirements.in | 1 + examples/modin_plugin/requirements.in | 1 + examples/snowflake_agent/requirements.in | 2 +- examples/snowflake_plugin/requirements.in | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/kfpytorch_plugin/requirements.in b/examples/kfpytorch_plugin/requirements.in index 0d91657c0..950dec72f 100644 --- a/examples/kfpytorch_plugin/requirements.in +++ b/examples/kfpytorch_plugin/requirements.in @@ -3,3 +3,4 @@ flytekitplugins-kfpytorch matplotlib torch tensorboardX +torchvision diff --git a/examples/modin_plugin/requirements.in b/examples/modin_plugin/requirements.in index bc60bba02..065912b20 100644 --- a/examples/modin_plugin/requirements.in +++ b/examples/modin_plugin/requirements.in @@ -6,3 +6,4 @@ flytekitplugins-modin scikit-learn modin ray +pandas \ No newline at end of file diff --git a/examples/snowflake_agent/requirements.in b/examples/snowflake_agent/requirements.in index 4f50a7b41..4c334f36d 100644 --- a/examples/snowflake_agent/requirements.in +++ b/examples/snowflake_agent/requirements.in @@ -1,3 +1,3 @@ flytekitplugins-snowflake==1.7.0 -flytekitplugins-envd==1.9.0 +flytekitplugins-envd==1.7.0 flytekit==1.7.1b1 diff --git a/examples/snowflake_plugin/requirements.in b/examples/snowflake_plugin/requirements.in index bbd72c042..b804a06af 100644 --- a/examples/snowflake_plugin/requirements.in +++ b/examples/snowflake_plugin/requirements.in @@ -1,3 +1,3 @@ flytekitplugins-snowflake==1.7.0 flytekit==1.7.1b1 -flytekitplugins-envd==1.9.0 +flytekitplugins-envd==1.7.0 From dd96f178f40972ff37f1d09954e070e700d5243a Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 17:59:05 +0800 Subject: [PATCH 29/47] test Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 2 +- examples/feast_integration/requirements.in | 2 +- requirements.in | 41 ---------------------- 3 files changed, 2 insertions(+), 43 deletions(-) delete mode 100644 requirements.in diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index e89cafc9f..53a93ea41 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -76,7 +76,7 @@ jobs: run: | pip freeze source .venv/bin/activate - pyflyte -vvv \ + pyflyte \ --pkgs ${{ matrix.example }} package \ --image mindmeld="ghcr.io/flyteorg/flytecookbook:core-latest" \ --image borebuster="ghcr.io/flyteorg/flytekit:py3.9-latest" \ diff --git a/examples/feast_integration/requirements.in b/examples/feast_integration/requirements.in index 0c72f86b3..373eedb91 100644 --- a/examples/feast_integration/requirements.in +++ b/examples/feast_integration/requirements.in @@ -1,4 +1,4 @@ -flytekit>=0.32.3 +flytekit wheel matplotlib flytekitplugins-deck-standard diff --git a/requirements.in b/requirements.in deleted file mode 100644 index 0b1b0257c..000000000 --- a/requirements.in +++ /dev/null @@ -1,41 +0,0 @@ -numpy<2.0.0 -flytekitplugins-airflow -flytekitplugins-athena -flytekitplugins-awsbatch -flytekitplugins-awssagemaker -flytekitplugins-bigquery -flytekitplugins-dask -flytekitplugins-dbt -flytekitplugins-deck-standard -flytekitplugins.dolt -flytekitplugins-duckdb -flytekitplugins-envd -flytekitplugins-great-expectations -flytekitplugins-hive -flytekitplugins-huggingface -flytekitplugins-pod -flytekitplugins-kfmpi -flytekitplugins-kfpytorch -flytekitplugins-kftensorflow -flytekitplugins-mlflow -flytekitplugins-mmcloud -flytekitplugins-modin -flytekitplugins-onnxpytorch -flytekitplugins-onnxscikitlearn -flytekitplugins-onnxtensorflow -flytekitplugins-openai -flytekitplugins-pandera -flytekitplugins-papermill -flytekitplugins-polars -flytekitplugins-pydantic -flytekitplugins-ray -flytekitplugins-snowflake -flytekitplugins-spark -flytekitplugins-sqlalchemy -flytekitplugins-vaex -flytekitplugins-whylogs -feast -xgboost -modin -dask[dataframe] -wandb From 21143ed5f34b98c1945aa3a07b476b323b4dca65 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 18:05:16 +0800 Subject: [PATCH 30/47] test Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 4 ++-- examples/feast_integration/requirements.in | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 53a93ea41..9635248ee 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -26,7 +26,7 @@ jobs: run: | echo "PACKAGES=$(find examples -maxdepth 1 -type d -exec basename '{}' \; \ | grep -v -e 'testing' -e 'examples' \ - | grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' \ + | grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' -e 'feast_integration' -e 'modin_plugin' \ | sort \ | jq --raw-input . \ | jq --slurp . \ @@ -81,4 +81,4 @@ jobs: --image mindmeld="ghcr.io/flyteorg/flytecookbook:core-latest" \ --image borebuster="ghcr.io/flyteorg/flytekit:py3.9-latest" \ --output spark-package.tgz \ - --force --fast + --force diff --git a/examples/feast_integration/requirements.in b/examples/feast_integration/requirements.in index 373eedb91..65bfffb97 100644 --- a/examples/feast_integration/requirements.in +++ b/examples/feast_integration/requirements.in @@ -5,4 +5,4 @@ flytekitplugins-deck-standard scikit-learn numpy boto3 -feast[aws]==0.25.0 +feast[aws] From 5a91a8c788228a148b6bbd56dfdf912329527afa Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 18:39:50 +0800 Subject: [PATCH 31/47] test Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 44 ++++++++++++++++++++++++++++---- scripts/serialize-example.sh | 2 +- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 9635248ee..5881cfcd2 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -26,20 +26,19 @@ jobs: run: | echo "PACKAGES=$(find examples -maxdepth 1 -type d -exec basename '{}' \; \ | grep -v -e 'testing' -e 'examples' \ - | grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' -e 'feast_integration' -e 'modin_plugin' \ + | grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' -e 'feast_integration' -e 'modin_plugin' -e 'sagemaker_inference_agent' \ | sort \ | jq --raw-input . \ | jq --slurp . \ | jq -c .)" >> "$GITHUB_OUTPUT" outputs: packages: "${{ steps.create-example-list.outputs.PACKAGES }}" - serialize_examples: + serialize: needs: [list_examples] runs-on: "ubuntu-latest" env: FLYTE_SDK_RICH_TRACEBACKS: "0" PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python - name: Serialize strategy: fail-fast: false matrix: @@ -74,11 +73,46 @@ jobs: - name: Pyflyte package working-directory: examples/${{ matrix.example }} run: | - pip freeze source .venv/bin/activate pyflyte \ --pkgs ${{ matrix.example }} package \ --image mindmeld="ghcr.io/flyteorg/flytecookbook:core-latest" \ --image borebuster="ghcr.io/flyteorg/flytekit:py3.9-latest" \ - --output spark-package.tgz \ + --output /root/flyte-package.tgz \ --force + tar -xvf flyte-package.tgz + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: snacks-${{ matrix.example }} + path: examples/${{ matrix.example }}/**/*.pb + + # Download all artifacts generated from the previous job. Startup a sandbox cluster then register all of them. + register: + name: Register example to sandbox + runs-on: ubuntu-latest + needs: [ "serialize" ] + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: "0" + - uses: unionai/flytectl-setup-action@v0.0.1 + - name: setup download artifact dir + run: | + mkdir download-artifact + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + path: ./download-artifact/ + - name: setup sandbox + run: | + flytectl demo start + flytectl config init + - name: Register examples + uses: unionai/flyte-register-action@v0.0.2 + with: + flytesnacks: false + proto: ./download-artifact/**/* + project: flytesnacks + version: "latest" + domain: development \ No newline at end of file diff --git a/scripts/serialize-example.sh b/scripts/serialize-example.sh index 57965cf64..5898a3ae6 100755 --- a/scripts/serialize-example.sh +++ b/scripts/serialize-example.sh @@ -17,7 +17,7 @@ serialize() { --image "$2" \ --image mindmeld="ghcr.io/flyteorg/flytecookbook:core-latest" \ --image borebuster="ghcr.io/flyteorg/flytekit:py3.9-latest" \ - --output /root/flyte-package.tgz \ + --output flyte-package.tgz \ --force } From 9ae8db3ac0ad70d15e0341d469d2b0c284adb256 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 18:40:16 +0800 Subject: [PATCH 32/47] test Signed-off-by: Kevin Su --- examples/kfpytorch_plugin/requirements.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kfpytorch_plugin/requirements.in b/examples/kfpytorch_plugin/requirements.in index 950dec72f..e398993b8 100644 --- a/examples/kfpytorch_plugin/requirements.in +++ b/examples/kfpytorch_plugin/requirements.in @@ -1,4 +1,4 @@ -flytekit==1.10.3 +flytekit flytekitplugins-kfpytorch matplotlib torch From c9e004724f58c9614e5ff2817ee0d859aa7c401b Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 18:43:43 +0800 Subject: [PATCH 33/47] tesat Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 5881cfcd2..5164fa4e8 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -78,7 +78,7 @@ jobs: --pkgs ${{ matrix.example }} package \ --image mindmeld="ghcr.io/flyteorg/flytecookbook:core-latest" \ --image borebuster="ghcr.io/flyteorg/flytekit:py3.9-latest" \ - --output /root/flyte-package.tgz \ + --output flyte-package.tgz \ --force tar -xvf flyte-package.tgz - name: Upload artifacts From 3c857ed89a4d3106112224cfd7d34e9a69438166 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 18:46:50 +0800 Subject: [PATCH 34/47] test Signed-off-by: Kevin Su --- _example_template/requirements.txt | 547 ---------------- examples/athena_plugin/requirements.txt | 198 ------ examples/aws_batch_plugin/requirements.txt | 186 ------ examples/blast/requirements.txt | 386 ----------- examples/databricks_agent/requirements.txt | 355 ----------- examples/databricks_plugin/requirements.txt | 355 ----------- examples/dbt_plugin/requirements.txt | 434 ------------- .../development_lifecycle/requirements.txt | 514 --------------- examples/dolt_plugin/requirements.txt | 310 --------- examples/duckdb_plugin/requirements.txt | 425 ------------- .../requirements.txt | 360 ----------- .../flyteinteractive_plugin/requirements.txt | 598 ------------------ examples/forecasting_sales/requirements.txt | 223 ------- .../greatexpectations_plugin/requirements.txt | 308 --------- .../house_price_prediction/requirements.txt | 309 --------- examples/k8s_dask_plugin/requirements.txt | 403 ------------ examples/k8s_pod_plugin/requirements.txt | 423 ------------- examples/mlflow_plugin/requirements.txt | 479 -------------- 18 files changed, 6813 deletions(-) delete mode 100644 _example_template/requirements.txt delete mode 100644 examples/athena_plugin/requirements.txt delete mode 100644 examples/aws_batch_plugin/requirements.txt delete mode 100644 examples/blast/requirements.txt delete mode 100644 examples/databricks_agent/requirements.txt delete mode 100644 examples/databricks_plugin/requirements.txt delete mode 100644 examples/dbt_plugin/requirements.txt delete mode 100644 examples/development_lifecycle/requirements.txt delete mode 100644 examples/dolt_plugin/requirements.txt delete mode 100644 examples/duckdb_plugin/requirements.txt delete mode 100644 examples/exploratory_data_analysis/requirements.txt delete mode 100644 examples/flyteinteractive_plugin/requirements.txt delete mode 100644 examples/forecasting_sales/requirements.txt delete mode 100644 examples/greatexpectations_plugin/requirements.txt delete mode 100644 examples/house_price_prediction/requirements.txt delete mode 100644 examples/k8s_dask_plugin/requirements.txt delete mode 100644 examples/k8s_pod_plugin/requirements.txt delete mode 100644 examples/mlflow_plugin/requirements.txt diff --git a/_example_template/requirements.txt b/_example_template/requirements.txt deleted file mode 100644 index d0334dfc9..000000000 --- a/_example_template/requirements.txt +++ /dev/null @@ -1,547 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.8 -# by the following command: -# -# pip-compile --resolver=backtracking requirements.in -# -adlfs==2023.4.0 - # via flytekit -aiobotocore==2.5.0 - # via s3fs -aiohttp==3.8.4 - # via - # adlfs - # aiobotocore - # gcsfs - # s3fs -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp -arrow==1.2.3 - # via jinja2-time -asttokens==2.2.1 - # via stack-data -async-timeout==4.0.2 - # via aiohttp -attrs==23.1.0 - # via - # aiohttp - # visions -azure-core==1.27.1 - # via - # adlfs - # azure-identity - # azure-storage-blob -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.13.0 - # via adlfs -azure-storage-blob==12.16.0 - # via adlfs -backcall==0.2.0 - # via ipython -binaryornot==0.4.4 - # via cookiecutter -botocore==1.29.76 - # via aiobotocore -cachetools==5.3.1 - # via google-auth -certifi==2023.5.7 - # via - # kubernetes - # requests -cffi==1.15.1 - # via - # azure-datalake-store - # cryptography -chardet==5.1.0 - # via binaryornot -charset-normalizer==3.1.0 - # via - # aiohttp - # requests -click==8.1.3 - # via - # cookiecutter - # flytekit - # rich-click -cloudpickle==2.2.1 - # via flytekit -comm==0.1.3 - # via ipykernel -contourpy==1.1.0 - # via matplotlib -cookiecutter==2.1.1 - # via flytekit -croniter==1.4.1 - # via flytekit -cryptography==41.0.1 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt - # pyopenssl - # secretstorage -cycler==0.11.0 - # via matplotlib -dacite==1.8.1 - # via ydata-profiling -dataclasses-json==0.5.8 - # via flytekit -debugpy==1.6.7 - # via ipykernel -decorator==5.1.1 - # via - # gcsfs - # ipython -deprecated==1.2.14 - # via flytekit -diskcache==5.6.1 - # via flytekit -docker==6.1.3 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.15 - # via flytekit -executing==1.2.0 - # via stack-data -flyteidl==1.5.11 - # via flytekit -flytekit==1.7.0 - # via - # -r requirements.in - # flytekitplugins-deck-standard -flytekitplugins-deck-standard==1.7.0 - # via -r requirements.in -fonttools==4.40.0 - # via matplotlib -frozenlist==1.3.3 - # via - # aiohttp - # aiosignal -fsspec==2023.6.0 - # via - # adlfs - # flytekit - # gcsfs - # s3fs -gcsfs==2023.6.0 - # via flytekit -gitdb==4.0.10 - # via gitpython -gitpython==3.1.35 - # via flytekit -google-api-core==2.11.1 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.20.0 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage - # kubernetes -google-auth-oauthlib==1.0.0 - # via gcsfs -google-cloud-core==2.3.2 - # via google-cloud-storage -google-cloud-storage==2.9.0 - # via gcsfs -google-crc32c==1.5.0 - # via google-resumable-media -google-resumable-media==2.5.0 - # via google-cloud-storage -googleapis-common-protos==1.59.1 - # via - # flyteidl - # flytekit - # google-api-core - # grpcio-status -grpcio==1.56.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.56.0 - # via flytekit -htmlmin==0.1.12 - # via ydata-profiling -idna==3.4 - # via - # requests - # yarl -imagehash==4.3.1 - # via - # visions - # ydata-profiling -importlib-metadata==6.7.0 - # via - # flytekit - # keyring -ipykernel==6.23.3 - # via ipywidgets -ipython==8.12.2 - # via - # ipykernel - # ipywidgets -ipywidgets==8.0.6 - # via flytekitplugins-deck-standard -isodate==0.6.1 - # via azure-storage-blob -jaraco-classes==3.2.3 - # via keyring -jedi==0.18.2 - # via ipython -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.2 - # via - # cookiecutter - # jinja2-time - # ydata-profiling -jinja2-time==0.2.0 - # via cookiecutter -jmespath==1.0.1 - # via botocore -joblib==1.2.0 - # via - # flytekit - # phik -jupyter-client==8.3.0 - # via ipykernel -jupyter-core==5.3.1 - # via - # ipykernel - # jupyter-client -jupyterlab-widgets==3.0.7 - # via ipywidgets -keyring==24.1.0 - # via flytekit -kiwisolver==1.4.4 - # via matplotlib -kubernetes==26.1.0 - # via flytekit -markdown==3.4.3 - # via flytekitplugins-deck-standard -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.3 - # via jinja2 -marshmallow==3.19.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -matplotlib==3.7.1 - # via - # -r requirements.in - # phik - # seaborn - # wordcloud - # ydata-profiling -matplotlib-inline==0.1.6 - # via - # ipykernel - # ipython -mdurl==0.1.2 - # via markdown-it-py -more-itertools==9.1.0 - # via jaraco-classes -msal==1.22.0 - # via - # azure-datalake-store - # azure-identity - # msal-extensions -msal-extensions==1.0.0 - # via azure-identity -multidict==6.0.4 - # via - # aiohttp - # yarl -multimethod==1.9.1 - # via - # visions - # ydata-profiling -mypy-extensions==1.0.0 - # via typing-inspect -natsort==8.4.0 - # via flytekit -nest-asyncio==1.5.6 - # via ipykernel -networkx==3.1 - # via visions -numpy==1.23.5 - # via - # contourpy - # flytekit - # imagehash - # matplotlib - # pandas - # patsy - # phik - # pyarrow - # pywavelets - # scipy - # seaborn - # statsmodels - # visions - # wordcloud - # ydata-profiling -oauthlib==3.2.2 - # via requests-oauthlib -packaging==23.1 - # via - # docker - # ipykernel - # marshmallow - # matplotlib - # plotly - # statsmodels -pandas==1.5.3 - # via - # flytekit - # phik - # seaborn - # statsmodels - # visions - # ydata-profiling -parso==0.8.3 - # via jedi -patsy==0.5.3 - # via statsmodels -pexpect==4.8.0 - # via ipython -phik==0.12.3 - # via ydata-profiling -pickleshare==0.7.5 - # via ipython -pillow==10.0.1 - # via - # imagehash - # matplotlib - # visions - # wordcloud -platformdirs==3.8.0 - # via jupyter-core -plotly==5.15.0 - # via flytekitplugins-deck-standard -portalocker==2.7.0 - # via msal-extensions -prompt-toolkit==3.0.38 - # via ipython -protobuf==4.23.3 - # via - # flyteidl - # google-api-core - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -psutil==5.9.5 - # via ipykernel -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data -pyarrow==10.0.1 - # via flytekit -pyasn1==0.5.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pycparser==2.21 - # via cffi -pydantic==1.10.9 - # via ydata-profiling -pygments==2.15.1 - # via - # ipython - # rich -pyjwt[crypto]==2.7.0 - # via msal -pyopenssl==23.2.0 - # via flytekit -pyparsing==3.1.0 - # via matplotlib -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # flytekit - # jupyter-client - # kubernetes - # matplotlib - # pandas -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2023.3 - # via - # flytekit - # pandas -pywavelets==1.4.1 - # via imagehash -pyyaml==6.0 - # via - # cookiecutter - # flytekit - # kubernetes - # responses - # ydata-profiling -pyzmq==25.1.0 - # via - # ipykernel - # jupyter-client -regex==2023.6.3 - # via docker-image-py -requests==2.31.0 - # via - # azure-core - # azure-datalake-store - # cookiecutter - # docker - # flytekit - # gcsfs - # google-api-core - # google-cloud-storage - # kubernetes - # msal - # requests-oauthlib - # responses - # ydata-profiling -requests-oauthlib==1.3.1 - # via - # google-auth-oauthlib - # kubernetes -responses==0.23.1 - # via flytekit -rich==13.4.2 - # via - # flytekit - # rich-click -rich-click==1.6.1 - # via flytekit -rsa==4.9 - # via google-auth -s3fs==2023.6.0 - # via flytekit -scipy==1.10.1 - # via - # imagehash - # phik - # statsmodels - # ydata-profiling -seaborn==0.12.2 - # via ydata-profiling -secretstorage==3.3.3 - # via keyring -six==1.16.0 - # via - # asttokens - # azure-core - # azure-identity - # google-auth - # isodate - # kubernetes - # patsy - # python-dateutil -smmap==5.0.0 - # via gitdb -sortedcontainers==2.4.0 - # via flytekit -stack-data==0.6.2 - # via ipython -statsd==3.3.0 - # via flytekit -statsmodels==0.14.0 - # via ydata-profiling -tangled-up-in-unicode==0.2.0 - # via visions -tenacity==8.2.2 - # via plotly -text-unidecode==1.3 - # via python-slugify -tornado==6.3.2 - # via - # ipykernel - # jupyter-client -tqdm==4.65.0 - # via ydata-profiling -traitlets==5.9.0 - # via - # comm - # ipykernel - # ipython - # ipywidgets - # jupyter-client - # jupyter-core - # matplotlib-inline -typeguard==2.13.3 - # via ydata-profiling -types-pyyaml==6.0.12.10 - # via responses -typing-extensions==4.6.3 - # via - # azure-core - # azure-storage-blob - # flytekit - # pydantic - # typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -urllib3==1.26.16 - # via - # botocore - # docker - # flytekit - # google-auth - # kubernetes - # requests - # responses -visions[type_image_path]==0.7.5 - # via ydata-profiling -wcwidth==0.2.6 - # via prompt-toolkit -websocket-client==1.6.1 - # via - # docker - # kubernetes -wheel==0.40.0 - # via - # -r requirements.in - # flytekit -widgetsnbextension==4.0.7 - # via ipywidgets -wordcloud==1.9.2 - # via ydata-profiling -wrapt==1.15.0 - # via - # aiobotocore - # deprecated - # flytekit -yarl==1.9.2 - # via aiohttp -ydata-profiling==4.3.1 - # via flytekitplugins-deck-standard -zipp==3.15.0 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/examples/athena_plugin/requirements.txt b/examples/athena_plugin/requirements.txt deleted file mode 100644 index c4bb4a72c..000000000 --- a/examples/athena_plugin/requirements.txt +++ /dev/null @@ -1,198 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# /Library/Developer/CommandLineTools/usr/bin/make requirements.txt -# -arrow==1.2.2 - # via jinja2-time -binaryornot==0.4.4 - # via cookiecutter -certifi==2021.10.8 - # via requests -cffi==1.15.0 - # via cryptography -chardet==4.0.0 - # via binaryornot -charset-normalizer==2.0.12 - # via requests -checksumdir==1.2.0 - # via flytekit -click==8.1.2 - # via - # cookiecutter - # flytekit -cloudpickle==2.0.0 - # via flytekit -cookiecutter==2.1.1 - # via flytekit -croniter==1.3.4 - # via flytekit -cryptography==37.0.3 - # via secretstorage -cycler==0.11.0 - # via matplotlib -dataclasses-json==0.5.7 - # via flytekit -decorator==5.1.1 - # via retry -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.13 - # via flytekit -flyteidl==0.24.21 - # via flytekit -flytekit==0.32.3 - # via - # -r ../../../common/requirements-common.in - # flytekitplugins-athena -flytekitplugins-athena==0.32.3 - # via -r requirements.in -fonttools==4.32.0 - # via matplotlib -googleapis-common-protos==1.56.0 - # via - # flyteidl - # grpcio-status -grpcio==1.44.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.44.0 - # via flytekit -idna==3.3 - # via requests -importlib-metadata==4.11.3 - # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.1 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 - # via cookiecutter -keyring==23.5.0 - # via flytekit -kiwisolver==1.4.2 - # via matplotlib -markupsafe==2.1.1 - # via jinja2 -marshmallow==3.15.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -matplotlib==3.5.1 - # via -r ../../../common/requirements-common.in -mypy-extensions==0.4.3 - # via typing-inspect -natsort==8.1.0 - # via flytekit -numpy==1.22.3 - # via - # matplotlib - # pandas - # pyarrow -packaging==21.3 - # via - # marshmallow - # matplotlib -pandas==1.4.2 - # via flytekit -pillow==9.1.0 - # via matplotlib -protobuf==3.20.0 - # via - # flyteidl - # flytekit - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -py==1.11.0 - # via retry -pyarrow==6.0.1 - # via flytekit -pycparser==2.21 - # via cffi -pyparsing==3.0.8 - # via - # matplotlib - # packaging -python-dateutil==2.8.2 - # via - # arrow - # croniter - # flytekit - # matplotlib - # pandas -python-json-logger==2.0.2 - # via flytekit -python-slugify==6.1.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.1 - # via - # flytekit - # pandas -pyyaml==6.0 - # via - # cookiecutter - # flytekit -regex==2022.3.15 - # via docker-image-py -requests==2.27.1 - # via - # cookiecutter - # flytekit - # responses -responses==0.20.0 - # via flytekit -retry==0.9.2 - # via flytekit -secretstorage==3.3.2 - # via keyring -six==1.16.0 - # via - # grpcio - # python-dateutil -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -typing-extensions==4.1.1 - # via - # flytekit - # typing-inspect -typing-inspect==0.7.1 - # via dataclasses-json -urllib3==1.26.9 - # via - # flytekit - # requests - # responses -wheel==0.37.1 - # via - # -r ../../../common/requirements-common.in - # flytekit -wrapt==1.14.0 - # via - # deprecated - # flytekit -zipp==3.8.0 - # via importlib-metadata diff --git a/examples/aws_batch_plugin/requirements.txt b/examples/aws_batch_plugin/requirements.txt deleted file mode 100644 index a1722ffb0..000000000 --- a/examples/aws_batch_plugin/requirements.txt +++ /dev/null @@ -1,186 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# /Library/Developer/CommandLineTools/usr/bin/make requirements.txt -# -arrow==1.2.2 - # via jinja2-time -binaryornot==0.4.4 - # via cookiecutter -certifi==2021.10.8 - # via requests -chardet==4.0.0 - # via binaryornot -charset-normalizer==2.0.12 - # via requests -checksumdir==1.2.0 - # via flytekit -click==8.1.2 - # via - # cookiecutter - # flytekit -cloudpickle==2.0.0 - # via flytekit -cookiecutter==1.7.3 - # via flytekit -croniter==1.3.4 - # via flytekit -cycler==0.11.0 - # via matplotlib -dataclasses-json==0.5.7 - # via flytekit -decorator==5.1.1 - # via retry -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.13 - # via flytekit -flyteidl==0.24.21 - # via flytekit -flytekit==0.32.3 - # via - # -r ../../../common/requirements-common.in - # flytekitplugins-awsbatch -flytekitplugins-awsbatch==0.32.3 - # via -r requirements.in -fonttools==4.32.0 - # via matplotlib -googleapis-common-protos==1.56.0 - # via - # flyteidl - # grpcio-status -grpcio==1.44.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.44.0 - # via flytekit -idna==3.3 - # via requests -importlib-metadata==4.11.3 - # via keyring -jinja2==3.1.1 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 - # via cookiecutter -keyring==23.5.0 - # via flytekit -kiwisolver==1.4.2 - # via matplotlib -markupsafe==2.1.1 - # via jinja2 -marshmallow==3.15.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -matplotlib==3.5.1 - # via -r ../../../common/requirements-common.in -mypy-extensions==0.4.3 - # via typing-inspect -natsort==8.1.0 - # via flytekit -numpy==1.22.3 - # via - # matplotlib - # pandas - # pyarrow -packaging==21.3 - # via - # marshmallow - # matplotlib -pandas==1.4.2 - # via flytekit -pillow==9.1.0 - # via matplotlib -poyo==0.5.0 - # via cookiecutter -protobuf==3.20.0 - # via - # flyteidl - # flytekit - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -py==1.11.0 - # via retry -pyarrow==6.0.1 - # via flytekit -pyparsing==3.0.8 - # via - # matplotlib - # packaging -python-dateutil==2.8.2 - # via - # croniter - # flytekit - # matplotlib - # pandas -python-json-logger==2.0.2 - # via flytekit -python-slugify==6.1.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.1 - # via - # flytekit - # pandas -pyyaml==6.0 - # via flytekit -regex==2022.3.15 - # via docker-image-py -requests==2.27.1 - # via - # cookiecutter - # flytekit - # responses -responses==0.20.0 - # via flytekit -retry==0.9.2 - # via flytekit -six==1.16.0 - # via - # cookiecutter - # grpcio - # python-dateutil -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -typing-extensions==4.1.1 - # via - # flytekit - # typing-inspect -typing-inspect==0.7.1 - # via dataclasses-json -urllib3==1.26.9 - # via - # flytekit - # requests - # responses -wheel==0.37.1 - # via - # -r ../../../common/requirements-common.in - # flytekit -wrapt==1.14.0 - # via - # deprecated - # flytekit -zipp==3.8.0 - # via importlib-metadata diff --git a/examples/blast/requirements.txt b/examples/blast/requirements.txt deleted file mode 100644 index c184f0158..000000000 --- a/examples/blast/requirements.txt +++ /dev/null @@ -1,386 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# /Library/Developer/CommandLineTools/usr/bin/make requirements.txt -# -arrow==1.2.3 - # via jinja2-time -asttokens==2.2.1 - # via stack-data -attrs==22.2.0 - # via visions -backcall==0.2.0 - # via ipython -binaryornot==0.4.4 - # via cookiecutter -certifi==2022.12.7 - # via requests -cffi==1.15.1 - # via cryptography -chardet==5.1.0 - # via binaryornot -charset-normalizer==3.0.1 - # via requests -click==8.1.3 - # via - # cookiecutter - # flytekit -cloudpickle==2.2.1 - # via flytekit -comm==0.1.2 - # via ipykernel -contourpy==1.0.7 - # via matplotlib -cookiecutter==2.1.1 - # via flytekit -croniter==1.3.8 - # via flytekit -cryptography==39.0.0 - # via - # pyopenssl - # secretstorage -cycler==0.11.0 - # via matplotlib -dataclasses-json==0.5.7 - # via flytekit -debugpy==1.6.6 - # via ipykernel -decorator==5.1.1 - # via - # ipython - # retry -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -docker==6.0.1 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.15 - # via flytekit -executing==1.2.0 - # via stack-data -flyteidl==1.3.5 - # via flytekit -flytekit==1.3.1 - # via - # -r requirements.in - # flytekitplugins-deck-standard -flytekitplugins-deck-standard==1.3.1 - # via -r requirements.in -fonttools==4.38.0 - # via matplotlib -gitdb==4.0.10 - # via gitpython -gitpython==3.1.30 - # via flytekit -googleapis-common-protos==1.58.0 - # via - # flyteidl - # flytekit - # grpcio-status -grpcio==1.51.1 - # via - # flytekit - # grpcio-status -grpcio-status==1.51.1 - # via flytekit -htmlmin==0.1.12 - # via ydata-profiling -idna==3.4 - # via requests -imagehash==4.3.1 - # via visions -importlib-metadata==6.0.0 - # via - # flytekit - # keyring -ipykernel==6.21.1 - # via ipywidgets -ipython==8.9.0 - # via - # ipykernel - # ipywidgets -ipywidgets==8.0.4 - # via flytekitplugins-deck-standard -jaraco-classes==3.2.3 - # via keyring -jedi==0.18.2 - # via ipython -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.2 - # via - # cookiecutter - # jinja2-time - # ydata-profiling -jinja2-time==0.2.0 - # via cookiecutter -joblib==1.2.0 - # via - # flytekit - # phik -jupyter-client==8.0.2 - # via ipykernel -jupyter-core==5.2.0 - # via - # ipykernel - # jupyter-client -jupyterlab-widgets==3.0.5 - # via ipywidgets -keyring==23.13.1 - # via flytekit -kiwisolver==1.4.4 - # via matplotlib -markdown==3.4.1 - # via flytekitplugins-deck-standard -markupsafe==2.1.2 - # via jinja2 -marshmallow==3.19.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -matplotlib==3.6.3 - # via - # -r requirements.in - # phik - # seaborn - # ydata-profiling -matplotlib-inline==0.1.6 - # via - # ipykernel - # ipython -more-itertools==9.0.0 - # via jaraco-classes -multimethod==1.9.1 - # via - # visions - # ydata-profiling -mypy-extensions==1.0.0 - # via typing-inspect -natsort==8.2.0 - # via flytekit -nest-asyncio==1.5.6 - # via ipykernel -networkx==3.0 - # via visions -numpy==1.23.5 - # via - # contourpy - # flytekit - # imagehash - # matplotlib - # pandas - # patsy - # phik - # pyarrow - # pywavelets - # scipy - # seaborn - # statsmodels - # visions - # ydata-profiling -packaging==23.0 - # via - # docker - # ipykernel - # marshmallow - # matplotlib - # statsmodels -pandas==1.5.3 - # via - # flytekit - # phik - # seaborn - # statsmodels - # visions - # ydata-profiling -pandas-profiling==3.6.6 - # via flytekitplugins-deck-standard -parso==0.8.3 - # via jedi -patsy==0.5.3 - # via statsmodels -pexpect==4.8.0 - # via ipython -phik==0.12.3 - # via ydata-profiling -pickleshare==0.7.5 - # via ipython -pillow==10.2.0 - # via - # imagehash - # matplotlib - # visions -platformdirs==3.0.0 - # via jupyter-core -plotly==5.13.0 - # via flytekitplugins-deck-standard -prompt-toolkit==3.0.36 - # via ipython -protobuf==4.21.12 - # via - # flyteidl - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -psutil==5.9.4 - # via ipykernel -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data -py==1.11.0 - # via retry -pyarrow==10.0.1 - # via flytekit -pycparser==2.21 - # via cffi -pydantic==1.10.4 - # via ydata-profiling -pygments==2.14.0 - # via ipython -pyopenssl==23.0.0 - # via flytekit -pyparsing==3.0.9 - # via matplotlib -python-dateutil==2.8.2 - # via - # arrow - # croniter - # flytekit - # jupyter-client - # matplotlib - # pandas -python-json-logger==2.0.4 - # via flytekit -python-slugify==8.0.0 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.7.1 - # via - # flytekit - # pandas -pywavelets==1.4.1 - # via imagehash -pyyaml==6.0 - # via - # cookiecutter - # flytekit - # ydata-profiling -pyzmq==25.0.0 - # via - # ipykernel - # jupyter-client -regex==2022.10.31 - # via docker-image-py -requests==2.28.2 - # via - # cookiecutter - # docker - # flytekit - # responses - # ydata-profiling -responses==0.22.0 - # via flytekit -retry==0.9.2 - # via flytekit -scipy==1.9.3 - # via - # imagehash - # phik - # statsmodels - # ydata-profiling -seaborn==0.12.2 - # via ydata-profiling -secretstorage==3.3.3 - # via keyring -six==1.16.0 - # via - # asttokens - # patsy - # python-dateutil -smmap==5.0.0 - # via gitdb -sortedcontainers==2.4.0 - # via flytekit -stack-data==0.6.2 - # via ipython -statsd==3.3.0 - # via flytekit -statsmodels==0.13.5 - # via ydata-profiling -tangled-up-in-unicode==0.2.0 - # via visions -tenacity==8.1.0 - # via plotly -text-unidecode==1.3 - # via python-slugify -toml==0.10.2 - # via responses -tornado==6.2 - # via - # ipykernel - # jupyter-client -tqdm==4.64.1 - # via ydata-profiling -traitlets==5.9.0 - # via - # comm - # ipykernel - # ipython - # ipywidgets - # jupyter-client - # jupyter-core - # matplotlib-inline -typeguard==2.13.3 - # via ydata-profiling -types-toml==0.10.8.3 - # via responses -typing-extensions==4.4.0 - # via - # flytekit - # pydantic - # typing-inspect -typing-inspect==0.8.0 - # via dataclasses-json -urllib3==1.26.14 - # via - # docker - # flytekit - # requests - # responses -visions[type-image-path]==0.7.5 - # via - # visions - # ydata-profiling -wcwidth==0.2.6 - # via prompt-toolkit -websocket-client==1.5.1 - # via docker -wheel==0.38.4 - # via - # -r requirements.in - # flytekit -widgetsnbextension==4.0.5 - # via ipywidgets -wrapt==1.14.1 - # via - # deprecated - # flytekit -ydata-profiling==4.0.0 - # via pandas-profiling -zipp==3.12.1 - # via importlib-metadata diff --git a/examples/databricks_agent/requirements.txt b/examples/databricks_agent/requirements.txt deleted file mode 100644 index 616c3aacb..000000000 --- a/examples/databricks_agent/requirements.txt +++ /dev/null @@ -1,355 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile requirements.in -# -adlfs==2023.8.0 - # via flytekit -aiobotocore==2.5.4 - # via s3fs -aiohttp==3.8.5 - # via - # adlfs - # aiobotocore - # gcsfs - # s3fs -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp -arrow==1.2.3 - # via cookiecutter -async-timeout==4.0.3 - # via aiohttp -attrs==23.1.0 - # via aiohttp -azure-core==1.29.1 - # via - # adlfs - # azure-identity - # azure-storage-blob -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.14.0 - # via adlfs -azure-storage-blob==12.17.0 - # via adlfs -binaryornot==0.4.4 - # via cookiecutter -botocore==1.31.17 - # via aiobotocore -cachetools==5.3.1 - # via google-auth -certifi==2023.7.22 - # via - # kubernetes - # requests -cffi==1.15.1 - # via - # azure-datalake-store - # cryptography -chardet==5.2.0 - # via binaryornot -charset-normalizer==3.2.0 - # via - # aiohttp - # requests -click==8.1.6 - # via - # cookiecutter - # flytekit - # rich-click -cloudpickle==2.2.1 - # via flytekit -cookiecutter==2.3.0 - # via flytekit -croniter==1.4.1 - # via flytekit -cryptography==41.0.6 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt - # pyopenssl - # secretstorage -dataclasses-json==0.5.9 - # via flytekit -decorator==5.1.1 - # via gcsfs -deprecated==1.2.14 - # via flytekit -diskcache==5.6.1 - # via flytekit -docker==6.1.3 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.15 - # via flytekit -flyteidl==1.5.15 - # via flytekit -flytekit==1.9.0 - # via flytekitplugins-spark -flytekitplugins-spark==1.9.0 - # via -r requirements.in -frozenlist==1.4.0 - # via - # aiohttp - # aiosignal -fsspec==2023.6.0 - # via - # adlfs - # flytekit - # gcsfs - # s3fs -gcsfs==2023.6.0 - # via flytekit -gitdb==4.0.10 - # via gitpython -gitpython==3.1.32 - # via flytekit -google-api-core==2.11.1 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.22.0 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage - # kubernetes -google-auth-oauthlib==1.0.0 - # via gcsfs -google-cloud-core==2.3.3 - # via google-cloud-storage -google-cloud-storage==2.10.0 - # via gcsfs -google-crc32c==1.5.0 - # via google-resumable-media -google-resumable-media==2.5.0 - # via google-cloud-storage -googleapis-common-protos==1.60.0 - # via - # flyteidl - # flytekit - # google-api-core - # grpcio-status -grpcio==1.53.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.53.0 - # via flytekit -idna==3.4 - # via - # requests - # yarl -importlib-metadata==6.8.0 - # via - # flytekit - # keyring -isodate==0.6.1 - # via azure-storage-blob -jaraco-classes==3.3.0 - # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.2 - # via cookiecutter -jmespath==1.0.1 - # via botocore -joblib==1.3.2 - # via flytekit -keyring==24.2.0 - # via flytekit -kubernetes==27.2.0 - # via flytekit -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.3 - # via jinja2 -marshmallow==3.20.1 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via - # dataclasses-json - # flytekit -marshmallow-jsonschema==0.13.0 - # via flytekit -mdurl==0.1.2 - # via markdown-it-py -more-itertools==10.1.0 - # via jaraco-classes -msal==1.23.0 - # via - # azure-datalake-store - # azure-identity - # msal-extensions -msal-extensions==1.0.0 - # via azure-identity -multidict==6.0.4 - # via - # aiohttp - # yarl -mypy-extensions==1.0.0 - # via typing-inspect -natsort==8.4.0 - # via flytekit -numpy==1.25.2 - # via - # flytekit - # pandas - # pyarrow -oauthlib==3.2.2 - # via - # kubernetes - # requests-oauthlib -packaging==23.1 - # via - # docker - # marshmallow -pandas==1.5.3 - # via flytekit -portalocker==2.7.0 - # via msal-extensions -protobuf==4.24.0 - # via - # flyteidl - # google-api-core - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -py4j==0.10.9.7 - # via pyspark -pyarrow==10.0.1 - # via flytekit -pyasn1==0.5.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pycparser==2.21 - # via cffi -pygments==2.16.1 - # via rich -pyjwt[crypto]==2.8.0 - # via - # msal - # pyjwt -pyopenssl==23.2.0 - # via flytekit -pyspark==3.4.1 - # via flytekitplugins-spark -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # flytekit - # kubernetes - # pandas -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2023.3 - # via - # flytekit - # pandas -pyyaml==6.0.1 - # via - # cookiecutter - # flytekit - # kubernetes -regex==2023.8.8 - # via docker-image-py -requests==2.31.0 - # via - # azure-core - # azure-datalake-store - # cookiecutter - # docker - # flytekit - # gcsfs - # google-api-core - # google-cloud-storage - # kubernetes - # msal - # requests-oauthlib -requests-oauthlib==1.3.1 - # via - # google-auth-oauthlib - # kubernetes -rich==13.5.2 - # via - # cookiecutter - # flytekit - # rich-click -rich-click==1.6.1 - # via flytekit -rsa==4.9 - # via google-auth -s3fs==2023.6.0 - # via flytekit -secretstorage==3.3.3 - # via keyring -six==1.16.0 - # via - # azure-core - # google-auth - # isodate - # kubernetes - # python-dateutil -smmap==5.0.0 - # via gitdb -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -typing-extensions==4.7.1 - # via - # azure-core - # azure-storage-blob - # flytekit - # typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -urllib3==1.26.17 - # via - # botocore - # docker - # flytekit - # google-auth - # kubernetes - # requests -websocket-client==1.6.1 - # via - # docker - # kubernetes -wheel==0.41.1 - # via flytekit -wrapt==1.15.0 - # via - # aiobotocore - # deprecated - # flytekit -yarl==1.9.2 - # via aiohttp -zipp==3.16.2 - # via importlib-metadata diff --git a/examples/databricks_plugin/requirements.txt b/examples/databricks_plugin/requirements.txt deleted file mode 100644 index 616c3aacb..000000000 --- a/examples/databricks_plugin/requirements.txt +++ /dev/null @@ -1,355 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile requirements.in -# -adlfs==2023.8.0 - # via flytekit -aiobotocore==2.5.4 - # via s3fs -aiohttp==3.8.5 - # via - # adlfs - # aiobotocore - # gcsfs - # s3fs -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp -arrow==1.2.3 - # via cookiecutter -async-timeout==4.0.3 - # via aiohttp -attrs==23.1.0 - # via aiohttp -azure-core==1.29.1 - # via - # adlfs - # azure-identity - # azure-storage-blob -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.14.0 - # via adlfs -azure-storage-blob==12.17.0 - # via adlfs -binaryornot==0.4.4 - # via cookiecutter -botocore==1.31.17 - # via aiobotocore -cachetools==5.3.1 - # via google-auth -certifi==2023.7.22 - # via - # kubernetes - # requests -cffi==1.15.1 - # via - # azure-datalake-store - # cryptography -chardet==5.2.0 - # via binaryornot -charset-normalizer==3.2.0 - # via - # aiohttp - # requests -click==8.1.6 - # via - # cookiecutter - # flytekit - # rich-click -cloudpickle==2.2.1 - # via flytekit -cookiecutter==2.3.0 - # via flytekit -croniter==1.4.1 - # via flytekit -cryptography==41.0.6 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt - # pyopenssl - # secretstorage -dataclasses-json==0.5.9 - # via flytekit -decorator==5.1.1 - # via gcsfs -deprecated==1.2.14 - # via flytekit -diskcache==5.6.1 - # via flytekit -docker==6.1.3 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.15 - # via flytekit -flyteidl==1.5.15 - # via flytekit -flytekit==1.9.0 - # via flytekitplugins-spark -flytekitplugins-spark==1.9.0 - # via -r requirements.in -frozenlist==1.4.0 - # via - # aiohttp - # aiosignal -fsspec==2023.6.0 - # via - # adlfs - # flytekit - # gcsfs - # s3fs -gcsfs==2023.6.0 - # via flytekit -gitdb==4.0.10 - # via gitpython -gitpython==3.1.32 - # via flytekit -google-api-core==2.11.1 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.22.0 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage - # kubernetes -google-auth-oauthlib==1.0.0 - # via gcsfs -google-cloud-core==2.3.3 - # via google-cloud-storage -google-cloud-storage==2.10.0 - # via gcsfs -google-crc32c==1.5.0 - # via google-resumable-media -google-resumable-media==2.5.0 - # via google-cloud-storage -googleapis-common-protos==1.60.0 - # via - # flyteidl - # flytekit - # google-api-core - # grpcio-status -grpcio==1.53.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.53.0 - # via flytekit -idna==3.4 - # via - # requests - # yarl -importlib-metadata==6.8.0 - # via - # flytekit - # keyring -isodate==0.6.1 - # via azure-storage-blob -jaraco-classes==3.3.0 - # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.2 - # via cookiecutter -jmespath==1.0.1 - # via botocore -joblib==1.3.2 - # via flytekit -keyring==24.2.0 - # via flytekit -kubernetes==27.2.0 - # via flytekit -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.3 - # via jinja2 -marshmallow==3.20.1 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via - # dataclasses-json - # flytekit -marshmallow-jsonschema==0.13.0 - # via flytekit -mdurl==0.1.2 - # via markdown-it-py -more-itertools==10.1.0 - # via jaraco-classes -msal==1.23.0 - # via - # azure-datalake-store - # azure-identity - # msal-extensions -msal-extensions==1.0.0 - # via azure-identity -multidict==6.0.4 - # via - # aiohttp - # yarl -mypy-extensions==1.0.0 - # via typing-inspect -natsort==8.4.0 - # via flytekit -numpy==1.25.2 - # via - # flytekit - # pandas - # pyarrow -oauthlib==3.2.2 - # via - # kubernetes - # requests-oauthlib -packaging==23.1 - # via - # docker - # marshmallow -pandas==1.5.3 - # via flytekit -portalocker==2.7.0 - # via msal-extensions -protobuf==4.24.0 - # via - # flyteidl - # google-api-core - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -py4j==0.10.9.7 - # via pyspark -pyarrow==10.0.1 - # via flytekit -pyasn1==0.5.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pycparser==2.21 - # via cffi -pygments==2.16.1 - # via rich -pyjwt[crypto]==2.8.0 - # via - # msal - # pyjwt -pyopenssl==23.2.0 - # via flytekit -pyspark==3.4.1 - # via flytekitplugins-spark -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # flytekit - # kubernetes - # pandas -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2023.3 - # via - # flytekit - # pandas -pyyaml==6.0.1 - # via - # cookiecutter - # flytekit - # kubernetes -regex==2023.8.8 - # via docker-image-py -requests==2.31.0 - # via - # azure-core - # azure-datalake-store - # cookiecutter - # docker - # flytekit - # gcsfs - # google-api-core - # google-cloud-storage - # kubernetes - # msal - # requests-oauthlib -requests-oauthlib==1.3.1 - # via - # google-auth-oauthlib - # kubernetes -rich==13.5.2 - # via - # cookiecutter - # flytekit - # rich-click -rich-click==1.6.1 - # via flytekit -rsa==4.9 - # via google-auth -s3fs==2023.6.0 - # via flytekit -secretstorage==3.3.3 - # via keyring -six==1.16.0 - # via - # azure-core - # google-auth - # isodate - # kubernetes - # python-dateutil -smmap==5.0.0 - # via gitdb -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -typing-extensions==4.7.1 - # via - # azure-core - # azure-storage-blob - # flytekit - # typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -urllib3==1.26.17 - # via - # botocore - # docker - # flytekit - # google-auth - # kubernetes - # requests -websocket-client==1.6.1 - # via - # docker - # kubernetes -wheel==0.41.1 - # via flytekit -wrapt==1.15.0 - # via - # aiobotocore - # deprecated - # flytekit -yarl==1.9.2 - # via aiohttp -zipp==3.16.2 - # via importlib-metadata diff --git a/examples/dbt_plugin/requirements.txt b/examples/dbt_plugin/requirements.txt deleted file mode 100644 index e15fb92e7..000000000 --- a/examples/dbt_plugin/requirements.txt +++ /dev/null @@ -1,434 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.8 -# by the following command: -# -# pip-compile --resolver=backtracking requirements.in -# -adlfs==2023.4.0 - # via flytekit -agate==1.6.3 - # via dbt-core -aiobotocore==2.5.2 - # via s3fs -aiohttp==3.8.5 - # via - # adlfs - # aiobotocore - # gcsfs - # s3fs -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp -arrow==1.2.3 - # via cookiecutter -async-timeout==4.0.2 - # via aiohttp -attrs==23.1.0 - # via - # aiohttp - # jsonschema -azure-core==1.28.0 - # via - # adlfs - # azure-identity - # azure-storage-blob -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.13.0 - # via adlfs -azure-storage-blob==12.17.0 - # via adlfs -babel==2.12.1 - # via agate -binaryornot==0.4.4 - # via cookiecutter -botocore==1.29.161 - # via aiobotocore -cachetools==5.3.1 - # via google-auth -certifi==2023.5.7 - # via - # kubernetes - # requests -cffi==1.15.1 - # via - # azure-datalake-store - # cryptography - # dbt-core -chardet==5.1.0 - # via binaryornot -charset-normalizer==3.2.0 - # via - # aiohttp - # requests -click==8.1.6 - # via - # cookiecutter - # dbt-core - # flytekit - # rich-click -cloudpickle==2.2.1 - # via flytekit -colorama==0.4.5 - # via dbt-core -cookiecutter==2.2.3 - # via flytekit -croniter==1.4.1 - # via flytekit -cryptography==41.0.2 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt - # pyopenssl -dataclasses-json==0.5.9 - # via flytekit -dbt-core==1.2.6 - # via - # dbt-postgres - # flytekitplugins-dbt -dbt-extractor==0.4.1 - # via dbt-core -dbt-postgres==1.2.6 - # via -r requirements.in -decorator==5.1.1 - # via gcsfs -deprecated==1.2.14 - # via flytekit -diskcache==5.6.1 - # via flytekit -docker==6.1.3 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.15 - # via flytekit -flyteidl==1.5.13 - # via flytekit -flytekit==1.8.1 - # via - # -r requirements.in - # flytekitplugins-dbt -flytekitplugins-dbt==1.8.1 - # via -r requirements.in -frozenlist==1.4.0 - # via - # aiohttp - # aiosignal -fsspec==2023.6.0 - # via - # adlfs - # flytekit - # gcsfs - # s3fs -future==0.18.3 - # via parsedatetime -gcsfs==2023.6.0 - # via flytekit -gitdb==4.0.10 - # via gitpython -gitpython==3.1.32 - # via flytekit -google-api-core==2.11.1 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.22.0 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage - # kubernetes -google-auth-oauthlib==1.0.0 - # via gcsfs -google-cloud-core==2.3.3 - # via google-cloud-storage -google-cloud-storage==2.10.0 - # via gcsfs -google-crc32c==1.5.0 - # via google-resumable-media -google-resumable-media==2.5.0 - # via google-cloud-storage -googleapis-common-protos==1.59.1 - # via - # flyteidl - # flytekit - # google-api-core - # grpcio-status -grpcio==1.56.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.56.0 - # via flytekit -hologram==0.0.15 - # via dbt-core -idna==3.4 - # via - # dbt-core - # requests - # yarl -importlib-metadata==6.8.0 - # via - # flytekit - # keyring -importlib-resources==6.0.0 - # via keyring -isodate==0.6.1 - # via - # agate - # azure-storage-blob - # dbt-core -jaraco-classes==3.3.0 - # via keyring -jinja2==2.11.3 - # via - # cookiecutter - # dbt-core -jmespath==1.0.1 - # via botocore -joblib==1.3.1 - # via flytekit -jsonschema==3.2.0 - # via hologram -keyring==24.2.0 - # via flytekit -kubernetes==27.2.0 - # via flytekit -leather==0.3.4 - # via agate -logbook==1.5.3 - # via dbt-core -markdown-it-py==3.0.0 - # via rich -markupsafe==2.0.1 - # via - # dbt-core - # jinja2 -marshmallow==3.19.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via - # dataclasses-json - # flytekit -marshmallow-jsonschema==0.13.0 - # via flytekit -mashumaro==2.9 - # via dbt-core -mdurl==0.1.2 - # via markdown-it-py -minimal-snowplow-tracker==0.0.2 - # via dbt-core -more-itertools==9.1.0 - # via jaraco-classes -msal==1.22.0 - # via - # azure-datalake-store - # azure-identity - # msal-extensions -msal-extensions==1.0.0 - # via azure-identity -msgpack==1.0.5 - # via mashumaro -multidict==6.0.4 - # via - # aiohttp - # yarl -mypy-extensions==1.0.0 - # via typing-inspect -natsort==8.4.0 - # via flytekit -networkx==2.8.8 - # via dbt-core -numpy==1.24.4 - # via - # flytekit - # pandas - # pyarrow -oauthlib==3.2.2 - # via - # kubernetes - # requests-oauthlib -packaging==21.3 - # via - # dbt-core - # docker - # marshmallow -pandas==1.5.3 - # via flytekit -parsedatetime==2.4 - # via agate -portalocker==2.7.0 - # via msal-extensions -protobuf==4.23.4 - # via - # dbt-core - # flyteidl - # google-api-core - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -psycopg2-binary==2.9.6 - # via dbt-postgres -pyarrow==10.0.1 - # via flytekit -pyasn1==0.5.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pycparser==2.21 - # via cffi -pygments==2.15.1 - # via rich -pyjwt[crypto]==2.8.0 - # via msal -pyopenssl==23.2.0 - # via flytekit -pyparsing==3.1.0 - # via packaging -pyrsistent==0.19.3 - # via jsonschema -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # flytekit - # hologram - # kubernetes - # pandas -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.1 - # via - # agate - # cookiecutter -pytimeparse==1.1.8 - # via - # agate - # flytekit -pytz==2023.3 - # via - # babel - # dbt-core - # flytekit - # pandas -pyyaml==5.4.1 - # via - # cookiecutter - # flytekit - # kubernetes - # mashumaro - # responses -regex==2023.6.3 - # via docker-image-py -requests==2.31.0 - # via - # azure-core - # azure-datalake-store - # cookiecutter - # dbt-core - # docker - # flytekit - # gcsfs - # google-api-core - # google-cloud-storage - # kubernetes - # minimal-snowplow-tracker - # msal - # requests-oauthlib - # responses -requests-oauthlib==1.3.1 - # via - # google-auth-oauthlib - # kubernetes -responses==0.23.1 - # via flytekit -rich==13.4.2 - # via - # flytekit - # rich-click -rich-click==1.6.1 - # via flytekit -rsa==4.9 - # via google-auth -s3fs==2023.6.0 - # via flytekit -six==1.16.0 - # via - # agate - # azure-core - # azure-identity - # google-auth - # isodate - # jsonschema - # kubernetes - # leather - # minimal-snowplow-tracker - # python-dateutil -smmap==5.0.0 - # via gitdb -sortedcontainers==2.4.0 - # via flytekit -sqlparse==0.4.3 - # via dbt-core -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -types-pyyaml==6.0.12.10 - # via responses -typing-extensions==4.7.1 - # via - # aioitertools - # azure-core - # azure-storage-blob - # dbt-core - # flytekit - # mashumaro - # rich - # typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -urllib3==1.26.16 - # via - # botocore - # docker - # flytekit - # google-auth - # kubernetes - # requests - # responses -websocket-client==1.6.1 - # via - # docker - # kubernetes -werkzeug==2.1.2 - # via dbt-core -wheel==0.40.0 - # via flytekit -wrapt==1.15.0 - # via - # aiobotocore - # deprecated - # flytekit -yarl==1.9.2 - # via aiohttp -zipp==3.16.2 - # via - # importlib-metadata - # importlib-resources - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/examples/development_lifecycle/requirements.txt b/examples/development_lifecycle/requirements.txt deleted file mode 100644 index 0daefe429..000000000 --- a/examples/development_lifecycle/requirements.txt +++ /dev/null @@ -1,514 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile requirements.in -# -adlfs==2023.10.0 - # via flytekit -aiobotocore==2.5.4 - # via s3fs -aiohttp==3.9.2 - # via - # adlfs - # aiobotocore - # gcsfs - # s3fs -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp -annotated-types==0.6.0 - # via pydantic -anyio==4.2.0 - # via azure-core -arrow==1.3.0 - # via cookiecutter -asttokens==2.4.1 - # via stack-data -attrs==23.2.0 - # via - # aiohttp - # visions -azure-core==1.29.6 - # via - # adlfs - # azure-identity - # azure-storage-blob -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.15.0 - # via adlfs -azure-storage-blob==12.19.0 - # via adlfs -binaryornot==0.4.4 - # via cookiecutter -botocore==1.31.17 - # via aiobotocore -cachetools==5.3.2 - # via google-auth -certifi==2023.11.17 - # via - # kubernetes - # requests -cffi==1.16.0 - # via - # azure-datalake-store - # cryptography -chardet==5.2.0 - # via binaryornot -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # cookiecutter - # flytekit - # rich-click -cloudpickle==3.0.0 - # via flytekit -comm==0.2.1 - # via ipywidgets -contourpy==1.2.0 - # via matplotlib -cookiecutter==2.5.0 - # via flytekit -croniter==2.0.1 - # via flytekit -cryptography==41.0.7 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt - # secretstorage -cycler==0.12.1 - # via matplotlib -dacite==1.8.1 - # via ydata-profiling -dataclasses-json==0.5.9 - # via flytekit -decorator==5.1.1 - # via - # gcsfs - # ipython -diskcache==5.6.3 - # via flytekit -docker==6.1.3 - # via flytekit -docstring-parser==0.15 - # via flytekit -envd==0.3.36 - # via flytekitplugins-envd -executing==2.0.1 - # via stack-data -flyteidl==1.10.6 - # via flytekit -flytekit==1.10.3b4 - # via - # -r requirements.in - # flytekitplugins-deck-standard - # flytekitplugins-envd -flytekitplugins-deck-standard==1.10.3b4 - # via -r requirements.in -flytekitplugins-envd==1.10.3b4 - # via -r requirements.in -fonttools==4.47.0 - # via matplotlib -frozenlist==1.4.1 - # via - # aiohttp - # aiosignal -fsspec==2023.9.2 - # via - # adlfs - # flytekit - # gcsfs - # s3fs -gcsfs==2023.9.2 - # via flytekit -google-api-core==2.15.0 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.26.1 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage - # kubernetes -google-auth-oauthlib==1.2.0 - # via gcsfs -google-cloud-core==2.4.1 - # via google-cloud-storage -google-cloud-storage==2.14.0 - # via gcsfs -google-crc32c==1.5.0 - # via - # google-cloud-storage - # google-resumable-media -google-resumable-media==2.7.0 - # via google-cloud-storage -googleapis-common-protos==1.62.0 - # via - # flyteidl - # flytekit - # google-api-core - # grpcio-status -grpcio==1.60.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.60.0 - # via flytekit -htmlmin==0.1.12 - # via ydata-profiling -idna==3.6 - # via - # anyio - # requests - # yarl -imagehash==4.3.1 - # via - # visions - # ydata-profiling -importlib-metadata==7.0.1 - # via - # flytekit - # keyring -ipython==8.19.0 - # via ipywidgets -ipywidgets==8.1.1 - # via flytekitplugins-deck-standard -isodate==0.6.1 - # via azure-storage-blob -jaraco-classes==3.3.0 - # via keyring -jedi==0.19.1 - # via ipython -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.2 - # via - # cookiecutter - # ydata-profiling -jmespath==1.0.1 - # via botocore -joblib==1.3.2 - # via - # flytekit - # phik - # scikit-learn -jsonpickle==3.0.2 - # via flytekit -jupyterlab-widgets==3.0.9 - # via ipywidgets -keyring==24.3.0 - # via flytekit -kiwisolver==1.4.5 - # via matplotlib -kubernetes==28.1.0 - # via flytekit -llvmlite==0.41.1 - # via numba -markdown==3.5.1 - # via flytekitplugins-deck-standard -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.3 - # via jinja2 -marshmallow==3.20.1 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via - # dataclasses-json - # flytekit -marshmallow-jsonschema==0.13.0 - # via flytekit -mashumaro==3.11 - # via flytekit -matplotlib==3.8.2 - # via - # phik - # seaborn - # wordcloud - # ydata-profiling -matplotlib-inline==0.1.6 - # via ipython -mdurl==0.1.2 - # via markdown-it-py -more-itertools==10.1.0 - # via jaraco-classes -msal==1.26.0 - # via - # azure-datalake-store - # azure-identity - # msal-extensions -msal-extensions==1.1.0 - # via azure-identity -multidict==6.0.4 - # via - # aiohttp - # yarl -multimethod==1.10 - # via - # visions - # ydata-profiling -mypy-extensions==1.0.0 - # via typing-inspect -networkx==3.2.1 - # via visions -numba==0.58.1 - # via ydata-profiling -numpy==1.25.2 - # via - # contourpy - # flytekit - # imagehash - # matplotlib - # numba - # pandas - # patsy - # phik - # pyarrow - # pywavelets - # scikit-learn - # scipy - # seaborn - # statsmodels - # visions - # wordcloud - # ydata-profiling -oauthlib==3.2.2 - # via - # kubernetes - # requests-oauthlib -packaging==23.2 - # via - # docker - # marshmallow - # matplotlib - # msal-extensions - # plotly - # statsmodels -pandas==1.5.3 - # via - # flytekit - # phik - # seaborn - # statsmodels - # visions - # ydata-profiling -parso==0.8.3 - # via jedi -patsy==0.5.5 - # via statsmodels -pexpect==4.9.0 - # via ipython -phik==0.12.3 - # via ydata-profiling -pillow==10.2.0 - # via - # imagehash - # matplotlib - # visions - # wordcloud -plotly==5.18.0 - # via - # -r requirements.in - # flytekitplugins-deck-standard -portalocker==2.8.2 - # via msal-extensions -prompt-toolkit==3.0.43 - # via ipython -protobuf==4.24.4 - # via - # flyteidl - # flytekit - # google-api-core - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data -pyarrow==14.0.2 - # via flytekit -pyasn1==0.5.1 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pycparser==2.21 - # via cffi -pydantic==2.5.3 - # via ydata-profiling -pydantic-core==2.14.6 - # via pydantic -pygments==2.17.2 - # via - # flytekitplugins-deck-standard - # ipython - # rich -pyjwt[crypto]==2.8.0 - # via - # msal - # pyjwt -pyparsing==3.1.1 - # via matplotlib -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # kubernetes - # matplotlib - # pandas -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2023.3.post1 - # via - # croniter - # pandas -pywavelets==1.5.0 - # via imagehash -pyyaml==6.0.1 - # via - # cookiecutter - # flytekit - # kubernetes - # ydata-profiling -requests==2.31.0 - # via - # azure-core - # azure-datalake-store - # cookiecutter - # docker - # flytekit - # gcsfs - # google-api-core - # google-cloud-storage - # kubernetes - # msal - # requests-oauthlib - # ydata-profiling -requests-oauthlib==1.3.1 - # via - # google-auth-oauthlib - # kubernetes -rich==13.7.0 - # via - # cookiecutter - # flytekit - # rich-click -rich-click==1.7.2 - # via flytekit -rsa==4.9 - # via google-auth -s3fs==2023.9.2 - # via flytekit -scikit-learn==1.3.2 - # via -r requirements.in -scipy==1.11.4 - # via - # imagehash - # phik - # scikit-learn - # statsmodels - # ydata-profiling -seaborn==0.12.2 - # via ydata-profiling -secretstorage==3.3.3 - # via keyring -six==1.16.0 - # via - # asttokens - # azure-core - # isodate - # kubernetes - # patsy - # python-dateutil -sniffio==1.3.0 - # via anyio -stack-data==0.6.3 - # via ipython -statsd==3.3.0 - # via flytekit -statsmodels==0.14.1 - # via ydata-profiling -tangled-up-in-unicode==0.2.0 - # via visions -tenacity==8.2.3 - # via plotly -text-unidecode==1.3 - # via python-slugify -threadpoolctl==3.2.0 - # via scikit-learn -tqdm==4.66.1 - # via ydata-profiling -traitlets==5.14.1 - # via - # comm - # ipython - # ipywidgets - # matplotlib-inline -typeguard==4.1.5 - # via ydata-profiling -types-python-dateutil==2.8.19.14 - # via arrow -typing-extensions==4.9.0 - # via - # azure-core - # azure-storage-blob - # flytekit - # mashumaro - # pydantic - # pydantic-core - # rich-click - # typeguard - # typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -urllib3==1.26.18 - # via - # botocore - # docker - # flytekit - # kubernetes - # requests -visions[type-image-path]==0.7.5 - # via - # visions - # ydata-profiling -wcwidth==0.2.12 - # via prompt-toolkit -websocket-client==1.7.0 - # via - # docker - # kubernetes -widgetsnbextension==4.0.9 - # via ipywidgets -wordcloud==1.9.3 - # via ydata-profiling -wrapt==1.16.0 - # via aiobotocore -yarl==1.9.4 - # via aiohttp -ydata-profiling==4.6.3 - # via flytekitplugins-deck-standard -zipp==3.17.0 - # via importlib-metadata diff --git a/examples/dolt_plugin/requirements.txt b/examples/dolt_plugin/requirements.txt deleted file mode 100644 index 5ea66fd63..000000000 --- a/examples/dolt_plugin/requirements.txt +++ /dev/null @@ -1,310 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# /Library/Developer/CommandLineTools/usr/bin/make requirements.txt -# -altair==4.2.0 - # via great-expectations -arrow==1.2.2 - # via jinja2-time -asttokens==2.0.5 - # via stack-data -attrs==21.4.0 - # via jsonschema -backcall==0.2.0 - # via ipython -binaryornot==0.4.4 - # via cookiecutter -certifi==2021.10.8 - # via requests -cffi==1.15.0 - # via cryptography -chardet==4.0.0 - # via binaryornot -charset-normalizer==2.0.12 - # via requests -checksumdir==1.2.0 - # via flytekit -click==8.1.2 - # via - # cookiecutter - # flytekit - # great-expectations -cloudpickle==2.0.0 - # via flytekit -colorama==0.4.4 - # via great-expectations -cookiecutter==2.1.1 - # via flytekit -croniter==1.3.4 - # via flytekit -cryptography==36.0.2 - # via - # great-expectations - # secretstorage -cycler==0.11.0 - # via matplotlib -dataclasses-json==0.5.7 - # via - # dolt-integrations - # flytekit -decorator==5.1.1 - # via - # ipython - # retry -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.13 - # via flytekit -dolt-integrations==0.1.5 - # via flytekitplugins-dolt -doltcli==0.1.17 - # via dolt-integrations -entrypoints==0.4 - # via altair -executing==0.8.3 - # via stack-data -fastjsonschema==2.15.3 - # via nbformat -flyteidl==0.24.21 - # via flytekit -flytekit==0.32.3 - # via - # -r ../../../common/requirements-common.in - # flytekitplugins-dolt -flytekitplugins-dolt==0.32.3 - # via -r requirements.in -fonttools==4.32.0 - # via matplotlib -googleapis-common-protos==1.56.0 - # via - # flyteidl - # grpcio-status -great-expectations==0.15.0 - # via -r requirements.in -grpcio==1.44.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.44.0 - # via flytekit -idna==3.3 - # via requests -importlib-metadata==4.11.3 - # via - # great-expectations - # keyring -ipython==8.2.0 - # via great-expectations -jedi==0.18.1 - # via ipython -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.0.3 - # via - # altair - # cookiecutter - # great-expectations - # jinja2-time -jinja2-time==0.2.0 - # via cookiecutter -jsonpatch==1.32 - # via great-expectations -jsonpointer==2.3 - # via jsonpatch -jsonschema==4.4.0 - # via - # altair - # great-expectations - # nbformat -jupyter-core==4.9.2 - # via nbformat -keyring==23.5.0 - # via flytekit -kiwisolver==1.4.2 - # via matplotlib -markupsafe==2.1.1 - # via jinja2 -marshmallow==3.15.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -matplotlib==3.5.1 - # via -r ../../../common/requirements-common.in -matplotlib-inline==0.1.3 - # via ipython -mistune==2.0.2 - # via great-expectations -mypy-extensions==0.4.3 - # via typing-inspect -natsort==8.1.0 - # via flytekit -nbformat==5.3.0 - # via great-expectations -numpy==1.22.3 - # via - # altair - # great-expectations - # matplotlib - # pandas - # pyarrow - # scipy -packaging==21.3 - # via - # great-expectations - # marshmallow - # matplotlib -pandas==1.4.2 - # via - # altair - # dolt-integrations - # flytekit - # great-expectations -parso==0.8.3 - # via jedi -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 - # via ipython -pillow==9.1.1 - # via matplotlib -prompt-toolkit==3.0.29 - # via ipython -protobuf==3.20.0 - # via - # flyteidl - # flytekit - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data -py==1.11.0 - # via retry -pyarrow==6.0.1 - # via flytekit -pycparser==2.21 - # via cffi -pygments==2.11.2 - # via ipython -pyparsing==2.4.7 - # via - # great-expectations - # matplotlib - # packaging -pyrsistent==0.18.1 - # via jsonschema -python-dateutil==2.8.2 - # via - # croniter - # flytekit - # great-expectations - # matplotlib - # pandas -python-json-logger==2.0.2 - # via flytekit -python-slugify==6.1.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.1 - # via - # flytekit - # great-expectations - # pandas -pytz-deprecation-shim==0.1.0.post0 - # via tzlocal -pyyaml==6.0 - # via - # cookiecutter - # flytekit -regex==2022.3.15 - # via docker-image-py -requests==2.27.1 - # via - # cookiecutter - # flytekit - # great-expectations - # responses -responses==0.20.0 - # via flytekit -retry==0.9.2 - # via flytekit -ruamel-yaml==0.17.17 - # via great-expectations -scipy==1.8.0 - # via great-expectations -secretstorage==3.3.2 - # via keyring -six==1.16.0 - # via - # grpcio - # python-dateutil -sortedcontainers==2.4.0 - # via flytekit -stack-data==0.2.0 - # via ipython -statsd==3.3.0 - # via flytekit -termcolor==1.1.0 - # via great-expectations -text-unidecode==1.3 - # via python-slugify -toolz==0.11.2 - # via altair -tqdm==4.64.0 - # via great-expectations -traitlets==5.1.1 - # via - # ipython - # jupyter-core - # matplotlib-inline - # nbformat -typing-extensions==4.1.1 - # via - # flytekit - # great-expectations - # typing-inspect -typing-inspect==0.7.1 - # via dataclasses-json -tzdata==2022.1 - # via pytz-deprecation-shim -tzlocal==4.2 - # via great-expectations -urllib3==1.26.9 - # via - # flytekit - # great-expectations - # requests - # responses -wcwidth==0.2.5 - # via prompt-toolkit -wheel==0.37.1 - # via - # -r ../../../common/requirements-common.in - # flytekit -wrapt==1.14.0 - # via - # deprecated - # flytekit -zipp==3.8.0 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/examples/duckdb_plugin/requirements.txt b/examples/duckdb_plugin/requirements.txt deleted file mode 100644 index f03472005..000000000 --- a/examples/duckdb_plugin/requirements.txt +++ /dev/null @@ -1,425 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# /Library/Developer/CommandLineTools/usr/bin/make requirements.txt -# -arrow==1.2.3 - # via jinja2-time -asttokens==2.2.1 - # via stack-data -attrs==22.2.0 - # via visions -backcall==0.2.0 - # via ipython -binaryornot==0.4.4 - # via cookiecutter -cachetools==5.3.0 - # via google-auth -certifi==2022.12.7 - # via - # kubernetes - # requests -cffi==1.15.1 - # via cryptography -chardet==5.1.0 - # via binaryornot -charset-normalizer==3.1.0 - # via requests -click==8.1.3 - # via - # cookiecutter - # flytekit -cloudpickle==2.2.1 - # via flytekit -comm==0.1.2 - # via ipykernel -contourpy==1.0.7 - # via matplotlib -cookiecutter==2.1.1 - # via flytekit -croniter==1.3.8 - # via flytekit -cryptography==39.0.2 - # via - # pyopenssl - # secretstorage -cycler==0.11.0 - # via matplotlib -dataclasses-json==0.5.7 - # via flytekit -debugpy==1.6.6 - # via ipykernel -decorator==5.1.1 - # via - # ipython - # retry -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -docker==6.0.1 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.15 - # via flytekit -duckdb==0.7.1 - # via flytekitplugins-duckdb -executing==1.2.0 - # via stack-data -flyteidl==1.3.9 - # via flytekit -flytekit==1.4.0 - # via - # -r requirements.in - # flytekitplugins-deck-standard - # flytekitplugins-duckdb -flytekitplugins-deck-standard==1.4.0 - # via -r requirements.in -flytekitplugins-duckdb==1.4.0 - # via -r requirements.in -fonttools==4.39.0 - # via matplotlib -gitdb==4.0.10 - # via gitpython -gitpython==3.1.31 - # via flytekit -google-auth==2.16.2 - # via kubernetes -googleapis-common-protos==1.58.0 - # via - # flyteidl - # flytekit - # grpcio-status -grpcio==1.51.3 - # via - # flytekit - # grpcio-status -grpcio-status==1.51.3 - # via flytekit -htmlmin==0.1.12 - # via ydata-profiling -idna==3.4 - # via requests -imagehash==4.3.1 - # via visions -importlib-metadata==6.0.0 - # via - # flytekit - # jupyter-client - # keyring - # markdown -ipykernel==6.21.3 - # via ipywidgets -ipython==8.11.0 - # via - # ipykernel - # ipywidgets -ipywidgets==8.0.4 - # via flytekitplugins-deck-standard -jaraco-classes==3.2.3 - # via keyring -jedi==0.18.2 - # via ipython -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.2 - # via - # cookiecutter - # jinja2-time - # ydata-profiling -jinja2-time==0.2.0 - # via cookiecutter -joblib==1.2.0 - # via - # flytekit - # phik -jupyter-client==8.0.3 - # via ipykernel -jupyter-core==5.2.0 - # via - # ipykernel - # jupyter-client -jupyterlab-widgets==3.0.5 - # via ipywidgets -keyring==23.13.1 - # via flytekit -kiwisolver==1.4.4 - # via matplotlib -kubernetes==26.1.0 - # via flytekit -markdown==3.4.1 - # via flytekitplugins-deck-standard -markupsafe==2.1.2 - # via jinja2 -marshmallow==3.19.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -matplotlib==3.6.3 - # via - # -r requirements.in - # phik - # seaborn - # ydata-profiling -matplotlib-inline==0.1.6 - # via - # ipykernel - # ipython -more-itertools==9.1.0 - # via jaraco-classes -multimethod==1.9.1 - # via - # visions - # ydata-profiling -mypy-extensions==1.0.0 - # via typing-inspect -natsort==8.3.1 - # via flytekit -nest-asyncio==1.5.6 - # via ipykernel -networkx==3.0 - # via visions -numpy==1.23.5 - # via - # contourpy - # flytekit - # imagehash - # matplotlib - # pandas - # patsy - # phik - # pyarrow - # pywavelets - # scipy - # seaborn - # statsmodels - # visions - # ydata-profiling -oauthlib==3.2.2 - # via requests-oauthlib -packaging==23.0 - # via - # docker - # ipykernel - # marshmallow - # matplotlib - # statsmodels -pandas==1.5.3 - # via - # flytekit - # phik - # seaborn - # statsmodels - # visions - # ydata-profiling -pandas-profiling==3.6.6 - # via flytekitplugins-deck-standard -parso==0.8.3 - # via jedi -patsy==0.5.3 - # via statsmodels -pexpect==4.8.0 - # via ipython -phik==0.12.3 - # via ydata-profiling -pickleshare==0.7.5 - # via ipython -pillow==10.2.0 - # via - # imagehash - # matplotlib - # visions -platformdirs==3.1.0 - # via jupyter-core -plotly==5.13.1 - # via flytekitplugins-deck-standard -prompt-toolkit==3.0.38 - # via ipython -protobuf==4.22.0 - # via - # flyteidl - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -psutil==5.9.4 - # via ipykernel -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data -py==1.11.0 - # via retry -pyarrow==10.0.1 - # via flytekit -pyasn1==0.4.8 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.2.8 - # via google-auth -pycparser==2.21 - # via cffi -pydantic==1.10.5 - # via ydata-profiling -pygments==2.14.0 - # via ipython -pyopenssl==23.0.0 - # via flytekit -pyparsing==3.0.9 - # via matplotlib -python-dateutil==2.8.2 - # via - # arrow - # croniter - # flytekit - # jupyter-client - # kubernetes - # matplotlib - # pandas -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.7.1 - # via - # flytekit - # pandas -pywavelets==1.4.1 - # via imagehash -pyyaml==6.0 - # via - # cookiecutter - # flytekit - # kubernetes - # ydata-profiling -pyzmq==25.0.0 - # via - # ipykernel - # jupyter-client -regex==2022.10.31 - # via docker-image-py -requests==2.28.2 - # via - # cookiecutter - # docker - # flytekit - # kubernetes - # requests-oauthlib - # responses - # ydata-profiling -requests-oauthlib==1.3.1 - # via kubernetes -responses==0.22.0 - # via flytekit -retry==0.9.2 - # via flytekit -rsa==4.9 - # via google-auth -scipy==1.9.3 - # via - # imagehash - # phik - # statsmodels - # ydata-profiling -seaborn==0.12.2 - # via ydata-profiling -secretstorage==3.3.3 - # via keyring -six==1.16.0 - # via - # asttokens - # google-auth - # kubernetes - # patsy - # python-dateutil -smmap==5.0.0 - # via gitdb -sortedcontainers==2.4.0 - # via flytekit -stack-data==0.6.2 - # via ipython -statsd==3.3.0 - # via flytekit -statsmodels==0.13.5 - # via ydata-profiling -tangled-up-in-unicode==0.2.0 - # via visions -tenacity==8.2.2 - # via plotly -text-unidecode==1.3 - # via python-slugify -toml==0.10.2 - # via responses -tornado==6.2 - # via - # ipykernel - # jupyter-client -tqdm==4.64.1 - # via ydata-profiling -traitlets==5.9.0 - # via - # comm - # ipykernel - # ipython - # ipywidgets - # jupyter-client - # jupyter-core - # matplotlib-inline -typeguard==2.13.3 - # via ydata-profiling -types-toml==0.10.8.5 - # via responses -typing-extensions==4.5.0 - # via - # flytekit - # pydantic - # typing-inspect -typing-inspect==0.8.0 - # via dataclasses-json -urllib3==1.26.14 - # via - # docker - # flytekit - # kubernetes - # requests - # responses -visions[type-image-path]==0.7.5 - # via - # visions - # ydata-profiling -wcwidth==0.2.6 - # via prompt-toolkit -websocket-client==1.5.1 - # via - # docker - # kubernetes -wheel==0.38.4 - # via - # -r requirements.in - # flytekit -widgetsnbextension==4.0.5 - # via ipywidgets -wrapt==1.15.0 - # via - # deprecated - # flytekit -ydata-profiling==4.0.0 - # via pandas-profiling -zipp==3.15.0 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/examples/exploratory_data_analysis/requirements.txt b/examples/exploratory_data_analysis/requirements.txt deleted file mode 100644 index 6695eaad8..000000000 --- a/examples/exploratory_data_analysis/requirements.txt +++ /dev/null @@ -1,360 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# /Library/Developer/CommandLineTools/usr/bin/make requirements.txt -# -ansiwrap==0.8.4 - # via papermill -arrow==1.2.2 - # via jinja2-time -asttokens==2.0.5 - # via stack-data -attrs==21.4.0 - # via jsonschema -backcall==0.2.0 - # via ipython -beautifulsoup4==4.11.1 - # via nbconvert -binaryornot==0.4.4 - # via cookiecutter -bleach==5.0.0 - # via nbconvert -certifi==2021.10.8 - # via requests -cffi==1.15.0 - # via cryptography -chardet==4.0.0 - # via binaryornot -charset-normalizer==2.0.12 - # via requests -checksumdir==1.2.0 - # via flytekit -click==8.1.2 - # via - # cookiecutter - # flytekit - # papermill -cloudpickle==2.0.0 - # via flytekit -cookiecutter==2.1.1 - # via flytekit -croniter==1.3.4 - # via flytekit -cryptography==37.0.3 - # via secretstorage -cycler==0.11.0 - # via matplotlib -dataclasses-json==0.5.7 - # via flytekit -debugpy==1.6.0 - # via ipykernel -decorator==5.1.1 - # via - # ipython - # retry -defusedxml==0.7.1 - # via nbconvert -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.13 - # via flytekit -entrypoints==0.4 - # via - # jupyter-client - # nbconvert - # papermill -executing==0.8.3 - # via stack-data -fastjsonschema==2.15.3 - # via nbformat -flyteidl==0.24.21 - # via flytekit -flytekit==0.32.3 - # via - # -r ../../../common/requirements-common.in - # flytekitplugins-papermill -flytekitplugins-papermill==0.32.3 - # via -r requirements.in -fonttools==4.32.0 - # via matplotlib -googleapis-common-protos==1.56.0 - # via - # flyteidl - # grpcio-status -grpcio==1.44.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.44.0 - # via flytekit -idna==3.3 - # via requests -importlib-metadata==4.11.3 - # via keyring -ipykernel==6.13.0 - # via flytekitplugins-papermill -ipython==8.2.0 - # via ipykernel -jedi==0.18.1 - # via ipython -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.1 - # via - # cookiecutter - # jinja2-time - # nbconvert -jinja2-time==0.2.0 - # via cookiecutter -joblib==1.1.0 - # via scikit-learn -jsonschema==4.4.0 - # via nbformat -jupyter-client==7.2.2 - # via - # ipykernel - # nbclient -jupyter-core==4.9.2 - # via - # jupyter-client - # nbconvert - # nbformat -jupyterlab-pygments==0.2.1 - # via nbconvert -keyring==23.5.0 - # via flytekit -kiwisolver==1.4.2 - # via matplotlib -markupsafe==2.1.1 - # via - # jinja2 - # nbconvert -marshmallow==3.15.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -matplotlib==3.5.1 - # via - # -r ../../../common/requirements-common.in - # -r requirements.in - # seaborn -matplotlib-inline==0.1.3 - # via - # ipykernel - # ipython -mistune==0.8.4 - # via nbconvert -mypy-extensions==0.4.3 - # via typing-inspect -natsort==8.1.0 - # via flytekit -nbclient==0.6.0 - # via - # nbconvert - # papermill -nbconvert==6.5.0 - # via flytekitplugins-papermill -nbformat==5.3.0 - # via - # nbclient - # nbconvert - # papermill -nest-asyncio==1.5.5 - # via - # ipykernel - # jupyter-client - # nbclient -numpy==1.22.3 - # via - # matplotlib - # pandas - # pyarrow - # scikit-learn - # scipy - # seaborn -packaging==21.3 - # via - # ipykernel - # marshmallow - # matplotlib - # nbconvert -pandas==1.4.2 - # via - # flytekit - # seaborn -pandocfilters==1.5.0 - # via nbconvert -papermill==2.3.4 - # via flytekitplugins-papermill -parso==0.8.3 - # via jedi -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 - # via ipython -pillow==9.1.0 - # via matplotlib -prompt-toolkit==3.0.29 - # via ipython -protobuf==3.20.0 - # via - # flyteidl - # flytekit - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -psutil==5.9.0 - # via ipykernel -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data -py==1.11.0 - # via retry -pyarrow==6.0.1 - # via flytekit -pycparser==2.21 - # via cffi -pygments==2.11.2 - # via - # ipython - # nbconvert -pyparsing==3.0.8 - # via - # matplotlib - # packaging -pyrsistent==0.18.1 - # via jsonschema -pysocks==1.7.1 - # via -r requirements.in -python-dateutil==2.8.2 - # via - # arrow - # croniter - # flytekit - # jupyter-client - # matplotlib - # pandas -python-json-logger==2.0.2 - # via flytekit -python-slugify==6.1.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.1 - # via - # flytekit - # pandas -pyyaml==6.0 - # via - # cookiecutter - # flytekit - # papermill -pyzmq==22.3.0 - # via jupyter-client -regex==2022.3.15 - # via docker-image-py -requests==2.27.1 - # via - # cookiecutter - # flytekit - # papermill - # responses -responses==0.20.0 - # via flytekit -retry==0.9.2 - # via flytekit -scikit-learn==1.0.2 - # via -r requirements.in -scipy==1.8.0 - # via - # scikit-learn - # seaborn -seaborn==0.11.2 - # via -r requirements.in -secretstorage==3.3.2 - # via keyring -six==1.16.0 - # via - # asttokens - # bleach - # grpcio - # python-dateutil -sortedcontainers==2.4.0 - # via flytekit -soupsieve==2.3.2 - # via beautifulsoup4 -stack-data==0.2.0 - # via ipython -statsd==3.3.0 - # via flytekit -tenacity==8.0.1 - # via papermill -text-unidecode==1.3 - # via python-slugify -textwrap3==0.9.2 - # via ansiwrap -threadpoolctl==3.1.0 - # via scikit-learn -tinycss2==1.1.1 - # via nbconvert -tornado==6.1 - # via - # ipykernel - # jupyter-client -tqdm==4.64.0 - # via papermill -traitlets==5.1.1 - # via - # ipykernel - # ipython - # jupyter-client - # jupyter-core - # matplotlib-inline - # nbclient - # nbconvert - # nbformat -typing-extensions==4.1.1 - # via - # flytekit - # typing-inspect -typing-inspect==0.7.1 - # via dataclasses-json -urllib3==1.26.9 - # via - # flytekit - # requests - # responses -wcwidth==0.2.5 - # via prompt-toolkit -webencodings==0.5.1 - # via - # bleach - # tinycss2 -wheel==0.37.1 - # via - # -r ../../../common/requirements-common.in - # flytekit -wrapt==1.14.0 - # via - # deprecated - # flytekit -zipp==3.8.0 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/examples/flyteinteractive_plugin/requirements.txt b/examples/flyteinteractive_plugin/requirements.txt deleted file mode 100644 index 8603b4fc2..000000000 --- a/examples/flyteinteractive_plugin/requirements.txt +++ /dev/null @@ -1,598 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# pip-compile requirements.in -# -adlfs==2023.10.0 - # via flytekit -aiobotocore==2.5.4 - # via s3fs -aiohttp==3.9.2 - # via - # adlfs - # aiobotocore - # gcsfs - # s3fs -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp -anyio==4.1.0 - # via jupyter-server -argon2-cffi==23.1.0 - # via jupyter-server -argon2-cffi-bindings==21.2.0 - # via argon2-cffi -arrow==1.3.0 - # via - # cookiecutter - # isoduration -asttokens==2.4.1 - # via stack-data -async-lru==2.0.4 - # via jupyterlab -async-timeout==4.0.3 - # via aiohttp -attrs==23.1.0 - # via - # aiohttp - # jsonschema - # referencing -azure-core==1.29.5 - # via - # adlfs - # azure-identity - # azure-storage-blob -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.15.0 - # via adlfs -azure-storage-blob==12.19.0 - # via adlfs -babel==2.13.1 - # via jupyterlab-server -beautifulsoup4==4.12.2 - # via nbconvert -binaryornot==0.4.4 - # via cookiecutter -bleach==6.1.0 - # via nbconvert -botocore==1.31.17 - # via aiobotocore -cachetools==5.3.2 - # via google-auth -certifi==2023.11.17 - # via requests -cffi==1.16.0 - # via - # argon2-cffi-bindings - # azure-datalake-store - # cryptography -chardet==5.2.0 - # via binaryornot -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # cookiecutter - # flytekit - # rich-click -cloudpickle==3.0.0 - # via flytekit -comm==0.2.0 - # via - # ipykernel - # ipywidgets -cookiecutter==2.5.0 - # via flytekit -croniter==2.0.1 - # via flytekit -cryptography==42.0.2 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt - # secretstorage -dataclasses-json==0.5.9 - # via flytekit -debugpy==1.8.0 - # via ipykernel -decorator==5.1.1 - # via - # gcsfs - # ipython -defusedxml==0.7.1 - # via nbconvert -diskcache==5.6.3 - # via flytekit -docker==6.1.3 - # via flytekit -docstring-parser==0.15 - # via flytekit -exceptiongroup==1.2.0 - # via - # anyio - # ipython -executing==2.0.1 - # via stack-data -fastjsonschema==2.19.0 - # via nbformat -flyteidl==1.10.7 - # via flytekit -flytekit==1.10.7 - # via - # -r requirements.in - # flytekitplugins-flyteinteractive -flytekitplugins-flyteinteractive==1.10.7 - # via -r requirements.in -fqdn==1.5.1 - # via jsonschema -frozenlist==1.4.0 - # via - # aiohttp - # aiosignal -fsspec==2023.9.2 - # via - # adlfs - # flytekit - # gcsfs - # s3fs -gcsfs==2023.9.2 - # via flytekit -google-api-core==2.15.0 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.25.1 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage -google-auth-oauthlib==1.1.0 - # via gcsfs -google-cloud-core==2.4.1 - # via google-cloud-storage -google-cloud-storage==2.13.0 - # via gcsfs -google-crc32c==1.5.0 - # via - # google-cloud-storage - # google-resumable-media -google-resumable-media==2.6.0 - # via google-cloud-storage -googleapis-common-protos==1.62.0 - # via - # flyteidl - # flytekit - # google-api-core - # grpcio-status - # protoc-gen-openapiv2 -grpcio==1.60.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.60.0 - # via flytekit -idna==3.6 - # via - # anyio - # jsonschema - # requests - # yarl -importlib-metadata==7.0.0 - # via - # flytekit - # jupyter-client - # jupyter-lsp - # jupyterlab - # jupyterlab-server - # keyring - # nbconvert -ipykernel==6.27.1 - # via - # jupyter - # jupyter-console - # jupyterlab - # qtconsole -ipython==8.18.1 - # via - # ipykernel - # ipywidgets - # jupyter-console -ipywidgets==8.1.1 - # via jupyter -isodate==0.6.1 - # via - # azure-storage-blob - # flytekit -isoduration==20.11.0 - # via jsonschema -jaraco-classes==3.3.0 - # via keyring -jedi==0.19.1 - # via ipython -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.2 - # via - # cookiecutter - # jupyter-server - # jupyterlab - # jupyterlab-server - # nbconvert -jmespath==1.0.1 - # via botocore -joblib==1.3.2 - # via flytekit -json5==0.9.14 - # via jupyterlab-server -jsonpickle==3.0.2 - # via flytekit -jsonpointer==2.4 - # via jsonschema -jsonschema[format-nongpl]==4.20.0 - # via - # jupyter-events - # jupyterlab-server - # nbformat -jsonschema-specifications==2023.11.2 - # via jsonschema -jupyter==1.0.0 - # via flytekitplugins-flyteinteractive -jupyter-client==8.6.0 - # via - # ipykernel - # jupyter-console - # jupyter-server - # nbclient - # qtconsole -jupyter-console==6.6.3 - # via jupyter -jupyter-core==5.5.0 - # via - # ipykernel - # jupyter-client - # jupyter-console - # jupyter-server - # jupyterlab - # nbclient - # nbconvert - # nbformat - # qtconsole -jupyter-events==0.9.0 - # via jupyter-server -jupyter-lsp==2.2.1 - # via jupyterlab -jupyter-server==2.12.1 - # via - # jupyter-lsp - # jupyterlab - # jupyterlab-server - # notebook - # notebook-shim -jupyter-server-terminals==0.4.4 - # via jupyter-server -jupyterlab==4.0.9 - # via notebook -jupyterlab-pygments==0.3.0 - # via nbconvert -jupyterlab-server==2.25.2 - # via - # jupyterlab - # notebook -jupyterlab-widgets==3.0.9 - # via ipywidgets -keyring==24.3.0 - # via flytekit -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.3 - # via - # jinja2 - # nbconvert -marshmallow==3.20.1 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via - # dataclasses-json - # flytekit -marshmallow-jsonschema==0.13.0 - # via flytekit -mashumaro==3.11 - # via flytekit -matplotlib-inline==0.1.6 - # via - # ipykernel - # ipython -mdurl==0.1.2 - # via markdown-it-py -mistune==3.0.2 - # via nbconvert -more-itertools==10.1.0 - # via jaraco-classes -msal==1.26.0 - # via - # azure-datalake-store - # azure-identity - # msal-extensions -msal-extensions==1.0.0 - # via azure-identity -multidict==6.0.4 - # via - # aiohttp - # yarl -mypy-extensions==1.0.0 - # via typing-inspect -nbclient==0.9.0 - # via nbconvert -nbconvert==7.12.0 - # via - # jupyter - # jupyter-server -nbformat==5.9.2 - # via - # jupyter-server - # nbclient - # nbconvert -nest-asyncio==1.5.8 - # via ipykernel -notebook==7.0.6 - # via jupyter -notebook-shim==0.2.3 - # via - # jupyterlab - # notebook -numpy==1.26.2 - # via pyarrow -oauthlib==3.2.2 - # via requests-oauthlib -overrides==7.4.0 - # via jupyter-server -packaging==23.2 - # via - # docker - # ipykernel - # jupyter-server - # jupyterlab - # jupyterlab-server - # marshmallow - # nbconvert - # qtconsole - # qtpy -pandocfilters==1.5.0 - # via nbconvert -parso==0.8.3 - # via jedi -pexpect==4.9.0 - # via ipython -platformdirs==4.1.0 - # via jupyter-core -portalocker==2.8.2 - # via msal-extensions -prometheus-client==0.19.0 - # via jupyter-server -prompt-toolkit==3.0.41 - # via - # ipython - # jupyter-console -protobuf==4.24.4 - # via - # flyteidl - # flytekit - # google-api-core - # googleapis-common-protos - # grpcio-status - # protoc-gen-openapiv2 -protoc-gen-openapiv2==0.0.1 - # via flyteidl -psutil==5.9.6 - # via ipykernel -ptyprocess==0.7.0 - # via - # pexpect - # terminado -pure-eval==0.2.2 - # via stack-data -pyarrow==14.0.1 - # via flytekit -pyasn1==0.5.1 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pycparser==2.21 - # via cffi -pygments==2.17.2 - # via - # ipython - # jupyter-console - # nbconvert - # qtconsole - # rich -pyjwt[crypto]==2.8.0 - # via - # msal - # pyjwt -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # jupyter-client -python-json-logger==2.0.7 - # via - # flytekit - # jupyter-events -python-slugify==8.0.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2023.3.post1 - # via croniter -pyyaml==6.0.1 - # via - # cookiecutter - # flytekit - # jupyter-events -pyzmq==25.1.2 - # via - # ipykernel - # jupyter-client - # jupyter-console - # jupyter-server - # qtconsole -qtconsole==5.5.1 - # via jupyter -qtpy==2.4.1 - # via qtconsole -referencing==0.32.0 - # via - # jsonschema - # jsonschema-specifications - # jupyter-events -requests==2.31.0 - # via - # azure-core - # azure-datalake-store - # cookiecutter - # docker - # flytekit - # gcsfs - # google-api-core - # google-cloud-storage - # jupyterlab-server - # msal - # requests-oauthlib -requests-oauthlib==1.3.1 - # via google-auth-oauthlib -rfc3339-validator==0.1.4 - # via - # jsonschema - # jupyter-events -rfc3986-validator==0.1.1 - # via - # jsonschema - # jupyter-events -rich==13.7.0 - # via - # cookiecutter - # flytekit - # rich-click -rich-click==1.7.2 - # via flytekit -rpds-py==0.13.2 - # via - # jsonschema - # referencing -rsa==4.9 - # via google-auth -s3fs==2023.9.2 - # via flytekit -secretstorage==3.3.3 - # via keyring -send2trash==1.8.2 - # via jupyter-server -six==1.16.0 - # via - # asttokens - # azure-core - # bleach - # isodate - # python-dateutil - # rfc3339-validator -sniffio==1.3.0 - # via anyio -soupsieve==2.5 - # via beautifulsoup4 -stack-data==0.6.3 - # via ipython -statsd==3.3.0 - # via flytekit -terminado==0.18.0 - # via - # jupyter-server - # jupyter-server-terminals -text-unidecode==1.3 - # via python-slugify -tinycss2==1.2.1 - # via nbconvert -tomli==2.0.1 - # via jupyterlab -tornado==6.4 - # via - # ipykernel - # jupyter-client - # jupyter-server - # jupyterlab - # notebook - # terminado -traitlets==5.14.0 - # via - # comm - # ipykernel - # ipython - # ipywidgets - # jupyter-client - # jupyter-console - # jupyter-core - # jupyter-events - # jupyter-server - # jupyterlab - # matplotlib-inline - # nbclient - # nbconvert - # nbformat - # qtconsole -types-python-dateutil==2.8.19.14 - # via arrow -typing-extensions==4.8.0 - # via - # aioitertools - # async-lru - # azure-core - # azure-storage-blob - # flytekit - # ipython - # mashumaro - # rich-click - # typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -uri-template==1.3.0 - # via jsonschema -urllib3==1.26.18 - # via - # botocore - # docker - # flytekit - # requests -wcwidth==0.2.12 - # via prompt-toolkit -webcolors==1.13 - # via jsonschema -webencodings==0.5.1 - # via - # bleach - # tinycss2 -websocket-client==1.7.0 - # via - # docker - # jupyter-server -widgetsnbextension==4.0.9 - # via ipywidgets -wrapt==1.16.0 - # via aiobotocore -yarl==1.9.4 - # via aiohttp -zipp==3.17.0 - # via importlib-metadata diff --git a/examples/forecasting_sales/requirements.txt b/examples/forecasting_sales/requirements.txt deleted file mode 100644 index 3f137afd8..000000000 --- a/examples/forecasting_sales/requirements.txt +++ /dev/null @@ -1,223 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# /Library/Developer/CommandLineTools/usr/bin/make requirements.txt -# -aiobotocore==2.2.0 - # via s3fs -aiohttp==3.8.1 - # via - # aiobotocore - # s3fs -aioitertools==0.10.0 - # via aiobotocore -aiosignal==1.2.0 - # via aiohttp -arrow==1.2.2 - # via jinja2-time -async-timeout==4.0.2 - # via aiohttp -attrs==21.4.0 - # via aiohttp -binaryornot==0.4.4 - # via cookiecutter -botocore==1.24.21 - # via aiobotocore -certifi==2021.10.8 - # via requests -cffi==1.15.1 - # via cryptography -chardet==4.0.0 - # via binaryornot -charset-normalizer==2.0.12 - # via - # aiohttp - # requests -checksumdir==1.2.0 - # via flytekit -click==8.1.2 - # via - # cookiecutter - # flytekit -cloudpickle==2.0.0 - # via flytekit -cookiecutter==2.1.1 - # via flytekit -croniter==1.3.4 - # via flytekit -cryptography==37.0.4 - # via secretstorage -dataclasses-json==0.5.7 - # via flytekit -decorator==5.1.1 - # via retry -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.13 - # via flytekit -flyteidl==0.24.21 - # via flytekit -flytekit==0.32.3 - # via flytekitplugins-spark -flytekitplugins-spark==0.32.3 - # via -r requirements.in -frozenlist==1.3.0 - # via - # aiohttp - # aiosignal -fsspec==2022.3.0 - # via s3fs -googleapis-common-protos==1.56.0 - # via - # flyteidl - # grpcio-status -grpcio==1.44.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.44.0 - # via flytekit -idna==3.3 - # via - # requests - # yarl -importlib-metadata==4.11.3 - # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.1 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 - # via cookiecutter -jmespath==1.0.0 - # via botocore -keyring==23.5.0 - # via flytekit -markupsafe==2.1.1 - # via jinja2 -marshmallow==3.15.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -multidict==6.0.2 - # via - # aiohttp - # yarl -mypy-extensions==0.4.3 - # via typing-inspect -natsort==8.1.0 - # via flytekit -numpy==1.22.3 - # via - # pandas - # pyarrow -packaging==21.3 - # via marshmallow -pandas==1.4.2 - # via flytekit -protobuf==3.20.0 - # via - # flyteidl - # flytekit - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -py==1.11.0 - # via retry -py4j==0.10.9 - # via pyspark -pyarrow==6.0.1 - # via flytekit -pycparser==2.21 - # via cffi -pyparsing==3.0.8 - # via packaging -pyspark==3.0.1 - # via - # -r requirements.in - # flytekitplugins-spark -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # flytekit - # pandas -python-json-logger==2.0.2 - # via flytekit -python-slugify==6.1.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.1 - # via - # flytekit - # pandas -pyyaml==6.0 - # via - # cookiecutter - # flytekit -regex==2022.3.15 - # via docker-image-py -requests==2.27.1 - # via - # cookiecutter - # flytekit - # responses -responses==0.20.0 - # via flytekit -retry==0.9.2 - # via flytekit -s3fs==2022.3.0 - # via -r requirements.in -secretstorage==3.3.2 - # via keyring -six==1.16.0 - # via - # grpcio - # python-dateutil -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -typing-extensions==4.1.1 - # via - # flytekit - # typing-inspect -typing-inspect==0.7.1 - # via dataclasses-json -urllib3==1.26.9 - # via - # botocore - # flytekit - # requests - # responses -wheel==0.37.1 - # via flytekit -wrapt==1.14.0 - # via - # aiobotocore - # deprecated - # flytekit -yarl==1.7.2 - # via aiohttp -zipp==3.8.0 - # via importlib-metadata diff --git a/examples/greatexpectations_plugin/requirements.txt b/examples/greatexpectations_plugin/requirements.txt deleted file mode 100644 index d4a6987f8..000000000 --- a/examples/greatexpectations_plugin/requirements.txt +++ /dev/null @@ -1,308 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# /Library/Developer/CommandLineTools/usr/bin/make requirements.txt -# -altair==4.2.0 - # via great-expectations -arrow==1.2.2 - # via jinja2-time -asttokens==2.0.5 - # via stack-data -attrs==21.4.0 - # via jsonschema -backcall==0.2.0 - # via ipython -binaryornot==0.4.4 - # via cookiecutter -certifi==2021.10.8 - # via requests -cffi==1.15.0 - # via cryptography -chardet==4.0.0 - # via binaryornot -charset-normalizer==2.0.12 - # via requests -checksumdir==1.2.0 - # via flytekit -click==8.1.2 - # via - # cookiecutter - # flytekit - # great-expectations -cloudpickle==2.0.0 - # via flytekit -colorama==0.4.4 - # via great-expectations -cookiecutter==2.1.1 - # via flytekit -croniter==1.3.4 - # via flytekit -cryptography==36.0.2 - # via - # great-expectations - # secretstorage -cycler==0.11.0 - # via matplotlib -dataclasses-json==0.5.7 - # via flytekit -decorator==5.1.1 - # via - # ipython - # retry -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.13 - # via flytekit -entrypoints==0.4 - # via altair -executing==0.8.3 - # via stack-data -fastjsonschema==2.15.3 - # via nbformat -flyteidl==0.24.21 - # via flytekit -flytekit==0.32.3 - # via - # -r ../../../common/requirements-common.in - # flytekitplugins-great-expectations -flytekitplugins-great-expectations==0.32.3 - # via -r requirements.in -fonttools==4.32.0 - # via matplotlib -googleapis-common-protos==1.56.0 - # via - # flyteidl - # grpcio-status -great-expectations==0.15.0 - # via flytekitplugins-great-expectations -greenlet==1.1.2 - # via sqlalchemy -grpcio==1.44.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.44.0 - # via flytekit -idna==3.3 - # via requests -importlib-metadata==4.11.3 - # via - # great-expectations - # keyring -ipython==8.2.0 - # via great-expectations -jedi==0.18.1 - # via ipython -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.0.3 - # via - # altair - # cookiecutter - # great-expectations - # jinja2-time -jinja2-time==0.2.0 - # via cookiecutter -jsonpatch==1.32 - # via great-expectations -jsonpointer==2.3 - # via jsonpatch -jsonschema==4.4.0 - # via - # altair - # great-expectations - # nbformat -jupyter-core==4.9.2 - # via nbformat -keyring==23.5.0 - # via flytekit -kiwisolver==1.4.2 - # via matplotlib -markupsafe==2.1.1 - # via jinja2 -marshmallow==3.15.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -matplotlib==3.5.1 - # via -r ../../../common/requirements-common.in -matplotlib-inline==0.1.3 - # via ipython -mistune==2.0.2 - # via great-expectations -mypy-extensions==0.4.3 - # via typing-inspect -natsort==8.1.0 - # via flytekit -nbformat==5.3.0 - # via great-expectations -numpy==1.22.3 - # via - # altair - # great-expectations - # matplotlib - # pandas - # pyarrow - # scipy -packaging==21.3 - # via - # great-expectations - # marshmallow - # matplotlib -pandas==1.4.2 - # via - # altair - # flytekit - # great-expectations -parso==0.8.3 - # via jedi -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 - # via ipython -pillow==9.1.0 - # via matplotlib -prompt-toolkit==3.0.29 - # via ipython -protobuf==3.20.0 - # via - # flyteidl - # flytekit - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data -py==1.11.0 - # via retry -pyarrow==6.0.1 - # via flytekit -pycparser==2.21 - # via cffi -pygments==2.11.2 - # via ipython -pyparsing==2.4.7 - # via - # great-expectations - # matplotlib - # packaging -pyrsistent==0.18.1 - # via jsonschema -python-dateutil==2.8.2 - # via - # arrow - # croniter - # flytekit - # great-expectations - # matplotlib - # pandas -python-json-logger==2.0.2 - # via flytekit -python-slugify==6.1.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.1 - # via - # flytekit - # great-expectations - # pandas -pytz-deprecation-shim==0.1.0.post0 - # via tzlocal -pyyaml==6.0 - # via - # cookiecutter - # flytekit -regex==2022.3.15 - # via docker-image-py -requests==2.27.1 - # via - # cookiecutter - # flytekit - # great-expectations - # responses -responses==0.20.0 - # via flytekit -retry==0.9.2 - # via flytekit -ruamel-yaml==0.17.17 - # via great-expectations -scipy==1.8.0 - # via great-expectations -secretstorage==3.3.2 - # via keyring -six==1.16.0 - # via - # grpcio - # python-dateutil -sortedcontainers==2.4.0 - # via flytekit -sqlalchemy==1.4.35 - # via flytekitplugins-great-expectations -stack-data==0.2.0 - # via ipython -statsd==3.3.0 - # via flytekit -termcolor==1.1.0 - # via great-expectations -text-unidecode==1.3 - # via python-slugify -toolz==0.11.2 - # via altair -tqdm==4.64.0 - # via great-expectations -traitlets==5.1.1 - # via - # ipython - # jupyter-core - # matplotlib-inline - # nbformat -typing-extensions==4.1.1 - # via - # flytekit - # great-expectations - # typing-inspect -typing-inspect==0.7.1 - # via dataclasses-json -tzdata==2022.1 - # via pytz-deprecation-shim -tzlocal==4.2 - # via great-expectations -urllib3==1.26.9 - # via - # flytekit - # great-expectations - # requests - # responses -wcwidth==0.2.5 - # via prompt-toolkit -wheel==0.37.1 - # via - # -r ../../../common/requirements-common.in - # flytekit -wrapt==1.14.0 - # via - # deprecated - # flytekit -zipp==3.8.0 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/examples/house_price_prediction/requirements.txt b/examples/house_price_prediction/requirements.txt deleted file mode 100644 index 6294a9a9f..000000000 --- a/examples/house_price_prediction/requirements.txt +++ /dev/null @@ -1,309 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# /Library/Developer/CommandLineTools/usr/bin/make requirements.txt -# -arrow==1.2.2 - # via jinja2-time -attrs==22.1.0 - # via visions -binaryornot==0.4.4 - # via cookiecutter -certifi==2021.10.8 - # via requests -cffi==1.15.1 - # via cryptography -chardet==4.0.0 - # via binaryornot -charset-normalizer==2.0.12 - # via requests -click==8.1.2 - # via - # cookiecutter - # flytekit -cloudpickle==2.0.0 - # via flytekit -cookiecutter==2.1.1 - # via flytekit -croniter==1.3.4 - # via flytekit -cryptography==38.0.1 - # via - # pyopenssl - # secretstorage -cycler==0.11.0 - # via matplotlib -dataclasses-json==0.5.7 - # via flytekit -decorator==5.1.1 - # via retry -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -docker==5.0.3 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.13 - # via flytekit -flyteidl==1.1.14 - # via flytekit -flytekit==1.1.1 - # via - # -r ../../../common/requirements-common.in - # flytekitplugins-deck-standard -flytekitplugins-deck-standard==1.1.1 - # via -r ../../../common/requirements-common.in -fonttools==4.32.0 - # via matplotlib -googleapis-common-protos==1.56.0 - # via - # flyteidl - # grpcio-status -grpcio==1.44.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.44.0 - # via flytekit -htmlmin==0.1.12 - # via pandas-profiling -idna==3.3 - # via requests -imagehash==4.3.0 - # via visions -importlib-metadata==4.11.3 - # via - # flytekit - # keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.1 - # via - # cookiecutter - # jinja2-time - # pandas-profiling -jinja2-time==0.2.0 - # via cookiecutter -joblib==1.1.0 - # via - # -r requirements.in - # pandas-profiling - # phik - # scikit-learn -keyring==23.5.0 - # via flytekit -kiwisolver==1.4.2 - # via matplotlib -markdown==3.4.1 - # via flytekitplugins-deck-standard -markupsafe==2.1.1 - # via jinja2 -marshmallow==3.15.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -matplotlib==3.5.1 - # via - # -r ../../../common/requirements-common.in - # -r requirements.in - # missingno - # pandas-profiling - # phik - # seaborn -missingno==0.5.1 - # via pandas-profiling -multimethod==1.8 - # via - # pandas-profiling - # visions -mypy-extensions==0.4.3 - # via typing-inspect -natsort==8.1.0 - # via flytekit -networkx==2.8.6 - # via visions -numpy==1.22.3 - # via - # imagehash - # matplotlib - # missingno - # pandas - # pandas-profiling - # patsy - # phik - # pyarrow - # scikit-learn - # scipy - # seaborn - # statsmodels - # visions - # xgboost -packaging==21.3 - # via - # marshmallow - # matplotlib - # statsmodels -pandas==1.4.2 - # via - # flytekit - # pandas-profiling - # phik - # seaborn - # statsmodels - # visions -pandas-profiling==3.3.0 - # via flytekitplugins-deck-standard -patsy==0.5.2 - # via statsmodels -phik==0.12.2 - # via pandas-profiling -pillow==9.1.0 - # via - # imagehash - # matplotlib - # visions -plotly==5.10.0 - # via flytekitplugins-deck-standard -protobuf==3.20.0 - # via - # flyteidl - # flytekit - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -py==1.11.0 - # via retry -pyarrow==6.0.1 - # via flytekit -pycparser==2.21 - # via cffi -pydantic==1.9.2 - # via pandas-profiling -pyopenssl==22.0.0 - # via flytekit -pyparsing==3.0.8 - # via - # matplotlib - # packaging -python-dateutil==2.8.2 - # via - # arrow - # croniter - # flytekit - # matplotlib - # pandas -python-json-logger==2.0.2 - # via flytekit -python-slugify==6.1.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.1 - # via - # flytekit - # pandas -pywavelets==1.3.0 - # via imagehash -pyyaml==6.0 - # via - # cookiecutter - # flytekit - # pandas-profiling -regex==2022.3.15 - # via docker-image-py -requests==2.27.1 - # via - # cookiecutter - # docker - # flytekit - # pandas-profiling - # responses -responses==0.20.0 - # via flytekit -retry==0.9.2 - # via flytekit -scikit-learn==1.0.2 - # via sklearn -scipy==1.8.0 - # via - # imagehash - # missingno - # pandas-profiling - # phik - # scikit-learn - # seaborn - # statsmodels - # xgboost -seaborn==0.11.2 - # via - # missingno - # pandas-profiling -secretstorage==3.3.3 - # via keyring -six==1.16.0 - # via - # grpcio - # patsy - # python-dateutil -sklearn==0.0 - # via -r requirements.in -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -statsmodels==0.13.2 - # via pandas-profiling -tabulate==0.8.9 - # via -r requirements.in -tangled-up-in-unicode==0.2.0 - # via - # pandas-profiling - # visions -tenacity==8.0.1 - # via plotly -text-unidecode==1.3 - # via python-slugify -threadpoolctl==3.1.0 - # via scikit-learn -tqdm==4.64.1 - # via pandas-profiling -typing-extensions==4.1.1 - # via - # flytekit - # pydantic - # typing-inspect -typing-inspect==0.7.1 - # via dataclasses-json -urllib3==1.26.9 - # via - # flytekit - # requests - # responses -visions[type_image_path]==0.7.5 - # via pandas-profiling -websocket-client==1.4.1 - # via docker -wheel==0.37.1 - # via - # -r ../../../common/requirements-common.in - # flytekit -wrapt==1.14.0 - # via - # deprecated - # flytekit -xgboost==1.5.2 - # via -r requirements.in -zipp==3.8.0 - # via importlib-metadata diff --git a/examples/k8s_dask_plugin/requirements.txt b/examples/k8s_dask_plugin/requirements.txt deleted file mode 100644 index 9bb2fbd80..000000000 --- a/examples/k8s_dask_plugin/requirements.txt +++ /dev/null @@ -1,403 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile requirements.in -# -adlfs==2023.8.0 - # via flytekit -aiobotocore==2.5.4 - # via s3fs -aiohttp==3.8.5 - # via - # adlfs - # aiobotocore - # gcsfs - # s3fs -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp -arrow==1.2.3 - # via cookiecutter -async-timeout==4.0.3 - # via aiohttp -attrs==23.1.0 - # via aiohttp -azure-core==1.29.3 - # via - # adlfs - # azure-identity - # azure-storage-blob -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.14.0 - # via adlfs -azure-storage-blob==12.17.0 - # via adlfs -binaryornot==0.4.4 - # via cookiecutter -botocore==1.31.17 - # via aiobotocore -cachetools==5.3.1 - # via google-auth -certifi==2023.7.22 - # via - # kubernetes - # requests -cffi==1.15.1 - # via - # azure-datalake-store - # cryptography -chardet==5.2.0 - # via binaryornot -charset-normalizer==3.2.0 - # via - # aiohttp - # requests -click==8.1.7 - # via - # cookiecutter - # dask - # distributed - # flytekit - # rich-click -cloudpickle==2.2.1 - # via - # dask - # distributed - # flytekit -cookiecutter==2.3.0 - # via flytekit -croniter==1.4.1 - # via flytekit -cryptography==41.0.6 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt - # pyopenssl - # secretstorage -dask[distributed]==2023.8.1 - # via - # dask - # distributed - # flytekitplugins-dask -dataclasses-json==0.5.9 - # via flytekit -decorator==5.1.1 - # via gcsfs -deprecated==1.2.14 - # via flytekit -diskcache==5.6.1 - # via flytekit -distributed==2023.8.1 - # via dask -docker==6.1.3 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.15 - # via flytekit -envd==0.3.39 - # via flytekitplugins-envd -flyteidl==1.5.16 - # via - # flytekit - # flytekitplugins-dask -flytekit==1.9.0 - # via - # flytekitplugins-dask - # flytekitplugins-envd -flytekitplugins-dask==1.9.0 - # via -r requirements.in -flytekitplugins-envd==1.9.0 - # via -r requirements.in -frozenlist==1.4.0 - # via - # aiohttp - # aiosignal -fsspec==2023.6.0 - # via - # adlfs - # dask - # flytekit - # gcsfs - # s3fs -gcsfs==2023.6.0 - # via flytekit -gitdb==4.0.10 - # via gitpython -gitpython==3.1.32 - # via flytekit -google-api-core==2.11.1 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.22.0 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage - # kubernetes -google-auth-oauthlib==1.0.0 - # via gcsfs -google-cloud-core==2.3.3 - # via google-cloud-storage -google-cloud-storage==2.10.0 - # via gcsfs -google-crc32c==1.5.0 - # via google-resumable-media -google-resumable-media==2.5.0 - # via google-cloud-storage -googleapis-common-protos==1.60.0 - # via - # flyteidl - # flytekit - # google-api-core - # grpcio-status -grpcio==1.53.0 - # via - # flytekit - # grpcio-status -grpcio-status==1.53.0 - # via flytekit -idna==3.4 - # via - # requests - # yarl -importlib-metadata==6.8.0 - # via - # dask - # flytekit - # keyring -isodate==0.6.1 - # via azure-storage-blob -jaraco-classes==3.3.0 - # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.2 - # via - # cookiecutter - # distributed -jmespath==1.0.1 - # via botocore -joblib==1.3.2 - # via flytekit -keyring==24.2.0 - # via flytekit -kubernetes==27.2.0 - # via flytekit -locket==1.0.0 - # via - # distributed - # partd -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.3 - # via jinja2 -marshmallow==3.20.1 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via - # dataclasses-json - # flytekit -marshmallow-jsonschema==0.13.0 - # via flytekit -mdurl==0.1.2 - # via markdown-it-py -more-itertools==10.1.0 - # via jaraco-classes -msal==1.23.0 - # via - # azure-datalake-store - # azure-identity - # msal-extensions -msal-extensions==1.0.0 - # via azure-identity -msgpack==1.0.5 - # via distributed -multidict==6.0.4 - # via - # aiohttp - # yarl -mypy-extensions==1.0.0 - # via typing-inspect -natsort==8.4.0 - # via flytekit -numpy==1.25.2 - # via - # flytekit - # pandas - # pyarrow -oauthlib==3.2.2 - # via - # kubernetes - # requests-oauthlib -packaging==23.1 - # via - # dask - # distributed - # docker - # marshmallow -pandas==1.5.3 - # via flytekit -partd==1.4.0 - # via dask -portalocker==2.7.0 - # via msal-extensions -protobuf==4.24.1 - # via - # flyteidl - # google-api-core - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -psutil==5.9.5 - # via distributed -pyarrow==10.0.1 - # via flytekit -pyasn1==0.5.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pycparser==2.21 - # via cffi -pygments==2.16.1 - # via rich -pyjwt[crypto]==2.8.0 - # via - # msal - # pyjwt -pyopenssl==23.2.0 - # via flytekit -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # flytekit - # kubernetes - # pandas -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.1 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2023.3 - # via - # flytekit - # pandas -pyyaml==6.0.1 - # via - # cookiecutter - # dask - # distributed - # flytekit - # kubernetes -regex==2023.8.8 - # via docker-image-py -requests==2.31.0 - # via - # azure-core - # azure-datalake-store - # cookiecutter - # docker - # flytekit - # gcsfs - # google-api-core - # google-cloud-storage - # kubernetes - # msal - # requests-oauthlib -requests-oauthlib==1.3.1 - # via - # google-auth-oauthlib - # kubernetes -rich==13.5.2 - # via - # cookiecutter - # flytekit - # rich-click -rich-click==1.6.1 - # via flytekit -rsa==4.9 - # via google-auth -s3fs==2023.6.0 - # via flytekit -secretstorage==3.3.3 - # via keyring -six==1.16.0 - # via - # azure-core - # google-auth - # isodate - # kubernetes - # python-dateutil -smmap==5.0.0 - # via gitdb -sortedcontainers==2.4.0 - # via - # distributed - # flytekit -statsd==3.3.0 - # via flytekit -tblib==2.0.0 - # via distributed -text-unidecode==1.3 - # via python-slugify -toolz==0.12.0 - # via - # dask - # distributed - # partd -tornado==6.3.3 - # via distributed -typing-extensions==4.7.1 - # via - # azure-core - # azure-storage-blob - # flytekit - # typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -urllib3==1.26.16 - # via - # botocore - # distributed - # docker - # flytekit - # google-auth - # kubernetes - # requests -websocket-client==1.6.2 - # via - # docker - # kubernetes -wheel==0.41.2 - # via flytekit -wrapt==1.15.0 - # via - # aiobotocore - # deprecated - # flytekit -yarl==1.9.2 - # via aiohttp -zict==3.0.0 - # via distributed -zipp==3.16.2 - # via importlib-metadata diff --git a/examples/k8s_pod_plugin/requirements.txt b/examples/k8s_pod_plugin/requirements.txt deleted file mode 100644 index f11d7902a..000000000 --- a/examples/k8s_pod_plugin/requirements.txt +++ /dev/null @@ -1,423 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.8 -# by the following command: -# -# /Library/Developer/CommandLineTools/usr/bin/make requirements.txt -# -appnope==0.1.3 - # via - # ipykernel - # ipython -arrow==1.2.3 - # via jinja2-time -asttokens==2.2.1 - # via stack-data -attrs==22.2.0 - # via visions -backcall==0.2.0 - # via ipython -binaryornot==0.4.4 - # via cookiecutter -cachetools==5.3.0 - # via google-auth -certifi==2022.12.7 - # via - # kubernetes - # requests -cffi==1.15.1 - # via cryptography -chardet==5.1.0 - # via binaryornot -charset-normalizer==3.0.1 - # via requests -click==8.1.3 - # via - # cookiecutter - # flytekit -cloudpickle==2.2.1 - # via flytekit -comm==0.1.2 - # via ipykernel -contourpy==1.0.7 - # via matplotlib -cookiecutter==2.1.1 - # via flytekit -croniter==1.3.8 - # via flytekit -cryptography==39.0.1 - # via pyopenssl -cycler==0.11.0 - # via matplotlib -dataclasses-json==0.5.7 - # via flytekit -debugpy==1.6.6 - # via ipykernel -decorator==5.1.1 - # via - # ipython - # retry -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -docker==6.0.1 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.15 - # via flytekit -executing==1.2.0 - # via stack-data -flyteidl==1.3.7 - # via flytekit -flytekit==1.3.2 - # via - # -r ../../../common/requirements-common.in - # flytekitplugins-deck-standard - # flytekitplugins-pod -flytekitplugins-deck-standard==1.3.2 - # via -r ../../../common/requirements-common.in -flytekitplugins-pod==1.3.2 - # via -r requirements.in -fonttools==4.38.0 - # via matplotlib -gitdb==4.0.10 - # via gitpython -gitpython==3.1.30 - # via flytekit -google-auth==2.16.0 - # via kubernetes -googleapis-common-protos==1.58.0 - # via - # flyteidl - # flytekit - # grpcio-status -grpcio==1.51.1 - # via - # flytekit - # grpcio-status -grpcio-status==1.51.1 - # via flytekit -htmlmin==0.1.12 - # via ydata-profiling -idna==3.4 - # via requests -imagehash==4.3.1 - # via visions -importlib-metadata==6.0.0 - # via - # flytekit - # jupyter-client - # keyring - # markdown -importlib-resources==5.10.2 - # via keyring -ipykernel==6.21.1 - # via ipywidgets -ipython==8.10.0 - # via - # ipykernel - # ipywidgets -ipywidgets==8.0.4 - # via flytekitplugins-deck-standard -jaraco-classes==3.2.3 - # via keyring -jedi==0.18.2 - # via ipython -jinja2==3.1.2 - # via - # cookiecutter - # jinja2-time - # ydata-profiling -jinja2-time==0.2.0 - # via cookiecutter -joblib==1.2.0 - # via - # flytekit - # phik -jupyter-client==8.0.2 - # via ipykernel -jupyter-core==5.2.0 - # via - # ipykernel - # jupyter-client -jupyterlab-widgets==3.0.5 - # via ipywidgets -keyring==23.13.1 - # via flytekit -kiwisolver==1.4.4 - # via matplotlib -kubernetes==25.3.0 - # via - # flytekit - # flytekitplugins-pod -markdown==3.4.1 - # via flytekitplugins-deck-standard -markupsafe==2.1.2 - # via jinja2 -marshmallow==3.19.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -matplotlib==3.6.3 - # via - # -r ../../../common/requirements-common.in - # phik - # seaborn - # ydata-profiling -matplotlib-inline==0.1.6 - # via - # ipykernel - # ipython -more-itertools==9.0.0 - # via jaraco-classes -multimethod==1.9.1 - # via - # visions - # ydata-profiling -mypy-extensions==1.0.0 - # via typing-inspect -natsort==8.2.0 - # via flytekit -nest-asyncio==1.5.6 - # via ipykernel -networkx==3.0 - # via visions -numpy==1.23.5 - # via - # contourpy - # flytekit - # imagehash - # matplotlib - # pandas - # patsy - # phik - # pyarrow - # pywavelets - # scipy - # seaborn - # statsmodels - # visions - # ydata-profiling -oauthlib==3.2.2 - # via requests-oauthlib -packaging==23.0 - # via - # docker - # ipykernel - # marshmallow - # matplotlib - # statsmodels -pandas==1.5.3 - # via - # flytekit - # phik - # seaborn - # statsmodels - # visions - # ydata-profiling -pandas-profiling==3.6.6 - # via flytekitplugins-deck-standard -parso==0.8.3 - # via jedi -patsy==0.5.3 - # via statsmodels -pexpect==4.8.0 - # via ipython -phik==0.12.3 - # via ydata-profiling -pickleshare==0.7.5 - # via ipython -pillow==9.4.0 - # via - # imagehash - # matplotlib - # visions -platformdirs==3.0.0 - # via jupyter-core -plotly==5.13.0 - # via flytekitplugins-deck-standard -prompt-toolkit==3.0.36 - # via ipython -protobuf==4.21.12 - # via - # flyteidl - # googleapis-common-protos - # grpcio-status - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl -psutil==5.9.4 - # via ipykernel -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data -py==1.11.0 - # via retry -pyarrow==10.0.1 - # via flytekit -pyasn1==0.4.8 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.2.8 - # via google-auth -pycparser==2.21 - # via cffi -pydantic==1.10.4 - # via ydata-profiling -pygments==2.14.0 - # via ipython -pyopenssl==23.0.0 - # via flytekit -pyparsing==3.0.9 - # via matplotlib -python-dateutil==2.8.2 - # via - # arrow - # croniter - # flytekit - # jupyter-client - # kubernetes - # matplotlib - # pandas -python-json-logger==2.0.4 - # via flytekit -python-slugify==8.0.0 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.7.1 - # via - # flytekit - # pandas -pywavelets==1.4.1 - # via imagehash -pyyaml==6.0 - # via - # cookiecutter - # flytekit - # kubernetes - # ydata-profiling -pyzmq==25.0.0 - # via - # ipykernel - # jupyter-client -regex==2022.10.31 - # via docker-image-py -requests==2.28.2 - # via - # cookiecutter - # docker - # flytekit - # kubernetes - # requests-oauthlib - # responses - # ydata-profiling -requests-oauthlib==1.3.1 - # via kubernetes -responses==0.22.0 - # via flytekit -retry==0.9.2 - # via flytekit -rsa==4.9 - # via google-auth -scipy==1.9.3 - # via - # imagehash - # phik - # statsmodels - # ydata-profiling -seaborn==0.12.2 - # via ydata-profiling -six==1.16.0 - # via - # asttokens - # google-auth - # kubernetes - # patsy - # python-dateutil -smmap==5.0.0 - # via gitdb -sortedcontainers==2.4.0 - # via flytekit -stack-data==0.6.2 - # via ipython -statsd==3.3.0 - # via flytekit -statsmodels==0.13.5 - # via ydata-profiling -tangled-up-in-unicode==0.2.0 - # via visions -tenacity==8.2.1 - # via plotly -text-unidecode==1.3 - # via python-slugify -toml==0.10.2 - # via responses -tornado==6.2 - # via - # ipykernel - # jupyter-client -tqdm==4.64.1 - # via ydata-profiling -traitlets==5.9.0 - # via - # comm - # ipykernel - # ipython - # ipywidgets - # jupyter-client - # jupyter-core - # matplotlib-inline -typeguard==2.13.3 - # via ydata-profiling -types-toml==0.10.8.3 - # via responses -typing-extensions==4.4.0 - # via - # flytekit - # pydantic - # typing-inspect -typing-inspect==0.8.0 - # via dataclasses-json -urllib3==1.26.14 - # via - # docker - # flytekit - # kubernetes - # requests - # responses -visions[type_image_path]==0.7.5 - # via ydata-profiling -wcwidth==0.2.6 - # via prompt-toolkit -websocket-client==1.5.1 - # via - # docker - # kubernetes -wheel==0.38.4 - # via - # -r ../../../common/requirements-common.in - # flytekit -widgetsnbextension==4.0.5 - # via ipywidgets -wrapt==1.14.1 - # via - # deprecated - # flytekit -ydata-profiling==4.0.0 - # via pandas-profiling -zipp==3.13.0 - # via - # importlib-metadata - # importlib-resources - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/examples/mlflow_plugin/requirements.txt b/examples/mlflow_plugin/requirements.txt deleted file mode 100644 index d6fde9c52..000000000 --- a/examples/mlflow_plugin/requirements.txt +++ /dev/null @@ -1,479 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile requirements.in -# -absl-py==2.1.0 - # via - # keras - # tensorboard - # tensorflow -adlfs==2024.2.0 - # via flytekit -aiobotocore==2.12.2 - # via s3fs -aiohttp==3.9.3 - # via - # adlfs - # aiobotocore - # gcsfs - # s3fs -aioitertools==0.11.0 - # via aiobotocore -aiosignal==1.3.1 - # via aiohttp -alembic==1.13.1 - # via mlflow -arrow==1.3.0 - # via cookiecutter -astunparse==1.6.3 - # via tensorflow -async-timeout==4.0.3 - # via aiohttp -attrs==23.2.0 - # via aiohttp -azure-core==1.30.1 - # via - # adlfs - # azure-identity - # azure-storage-blob -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.15.0 - # via adlfs -azure-storage-blob==12.19.1 - # via adlfs -backports-tarfile==1.0.0 - # via jaraco-context -binaryornot==0.4.4 - # via cookiecutter -blinker==1.7.0 - # via flask -botocore==1.34.51 - # via aiobotocore -cachetools==5.3.3 - # via google-auth -certifi==2024.2.2 - # via requests -cffi==1.16.0 - # via - # azure-datalake-store - # cryptography -chardet==5.2.0 - # via binaryornot -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # cookiecutter - # databricks-cli - # flask - # flytekit - # mlflow - # rich-click -cloudpickle==2.2.1 - # via - # flytekit - # mlflow -cookiecutter==2.6.0 - # via flytekit -croniter==2.0.3 - # via flytekit -cryptography==42.0.5 - # via - # azure-identity - # azure-storage-blob - # msal - # pyjwt -databricks-cli==0.18.0 - # via mlflow -dataclasses-json==0.5.9 - # via flytekit -decorator==5.1.1 - # via gcsfs -diskcache==5.6.3 - # via flytekit -docker==6.1.3 - # via - # flytekit - # mlflow -docstring-parser==0.16 - # via flytekit -entrypoints==0.4 - # via mlflow -flask==2.3.3 - # via - # mlflow - # prometheus-flask-exporter -flatbuffers==24.3.25 - # via tensorflow -flyteidl==1.11.1b1 - # via flytekit -flytekit==1.11.0 - # via flytekitplugins-mlflow -flytekitplugins-mlflow==1.11.0 - # via -r requirements.in -frozenlist==1.4.1 - # via - # aiohttp - # aiosignal -fsspec==2024.3.1 - # via - # adlfs - # flytekit - # gcsfs - # s3fs -gast==0.5.4 - # via tensorflow -gcsfs==2024.3.1 - # via flytekit -gitdb==4.0.11 - # via gitpython -gitpython==3.1.43 - # via mlflow -google-api-core==2.18.0 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.29.0 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage -google-auth-oauthlib==1.2.0 - # via gcsfs -google-cloud-core==2.4.1 - # via google-cloud-storage -google-cloud-storage==2.16.0 - # via gcsfs -google-crc32c==1.5.0 - # via - # google-cloud-storage - # google-resumable-media -google-pasta==0.2.0 - # via tensorflow -google-resumable-media==2.7.0 - # via google-cloud-storage -googleapis-common-protos==1.63.0 - # via - # flyteidl - # flytekit - # google-api-core - # grpcio-status - # protoc-gen-openapiv2 -grpcio==1.62.1 - # via - # flytekit - # grpcio-status - # tensorboard - # tensorflow -grpcio-status==1.62.1 - # via flytekit -gunicorn==20.1.0 - # via mlflow -h5py==3.10.0 - # via - # keras - # tensorflow -idna==3.6 - # via - # requests - # yarl -importlib-metadata==5.2.0 - # via - # flytekit - # keyring - # mlflow -isodate==0.6.1 - # via - # azure-storage-blob - # flytekit -itsdangerous==2.1.2 - # via flask -jaraco-classes==3.4.0 - # via keyring -jaraco-context==5.3.0 - # via keyring -jaraco-functools==4.0.0 - # via keyring -jinja2==3.1.3 - # via - # cookiecutter - # flask -jmespath==1.0.1 - # via botocore -joblib==1.4.0 - # via flytekit -jsonpickle==3.0.3 - # via flytekit -keras==3.2.0 - # via tensorflow -keyring==25.1.0 - # via flytekit -libclang==18.1.1 - # via tensorflow -mako==1.3.2 - # via alembic -markdown==3.6 - # via tensorboard -markdown-it-py==3.0.0 - # via - # flytekit - # rich -markupsafe==2.1.5 - # via - # jinja2 - # mako - # werkzeug -marshmallow==3.21.1 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via - # dataclasses-json - # flytekit -marshmallow-jsonschema==0.13.0 - # via flytekit -mashumaro==3.12 - # via flytekit -mdurl==0.1.2 - # via markdown-it-py -ml-dtypes==0.3.2 - # via - # keras - # tensorflow -mlflow==1.30.1 - # via flytekitplugins-mlflow -more-itertools==10.2.0 - # via - # jaraco-classes - # jaraco-functools -msal==1.28.0 - # via - # azure-datalake-store - # azure-identity - # msal-extensions -msal-extensions==1.1.0 - # via azure-identity -multidict==6.0.5 - # via - # aiohttp - # yarl -mypy-extensions==1.0.0 - # via typing-inspect -namex==0.0.7 - # via keras -numpy==1.26.4 - # via - # h5py - # keras - # ml-dtypes - # mlflow - # opt-einsum - # pandas - # pyarrow - # scipy - # tensorboard - # tensorflow -oauthlib==3.2.2 - # via - # databricks-cli - # requests-oauthlib -opt-einsum==3.3.0 - # via tensorflow -optree==0.11.0 - # via keras -packaging==21.3 - # via - # docker - # marshmallow - # mlflow - # msal-extensions - # plotly - # tensorflow -pandas==1.5.3 - # via - # flytekitplugins-mlflow - # mlflow -plotly==5.20.0 - # via flytekitplugins-mlflow -portalocker==2.8.2 - # via msal-extensions -prometheus-client==0.20.0 - # via prometheus-flask-exporter -prometheus-flask-exporter==0.23.0 - # via mlflow -proto-plus==1.23.0 - # via google-api-core -protobuf==4.25.3 - # via - # flyteidl - # flytekit - # google-api-core - # googleapis-common-protos - # grpcio-status - # mlflow - # proto-plus - # protoc-gen-openapiv2 - # tensorboard - # tensorflow -protoc-gen-openapiv2==0.0.1 - # via flyteidl -pyarrow==15.0.2 - # via flytekit -pyasn1==0.6.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.4.0 - # via google-auth -pycparser==2.22 - # via cffi -pygments==2.17.2 - # via - # flytekit - # rich -pyjwt[crypto]==2.8.0 - # via - # databricks-cli - # msal -pyparsing==3.1.2 - # via packaging -python-dateutil==2.9.0.post0 - # via - # arrow - # botocore - # croniter - # pandas -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.4 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.7.1 - # via - # croniter - # mlflow - # pandas -pyyaml==6.0.1 - # via - # cookiecutter - # flytekit - # mlflow -querystring-parser==1.2.4 - # via mlflow -requests==2.31.0 - # via - # azure-core - # azure-datalake-store - # cookiecutter - # databricks-cli - # docker - # flytekit - # gcsfs - # google-api-core - # google-cloud-storage - # mlflow - # msal - # requests-oauthlib - # tensorflow -requests-oauthlib==2.0.0 - # via google-auth-oauthlib -rich==13.7.1 - # via - # cookiecutter - # flytekit - # keras - # rich-click -rich-click==1.7.4 - # via flytekit -rsa==4.9 - # via google-auth -s3fs==2024.3.1 - # via flytekit -scipy==1.13.0 - # via mlflow -six==1.16.0 - # via - # astunparse - # azure-core - # databricks-cli - # google-pasta - # isodate - # python-dateutil - # querystring-parser - # tensorboard - # tensorflow -smmap==5.0.1 - # via gitdb -sqlalchemy==1.4.52 - # via - # alembic - # mlflow -sqlparse==0.4.4 - # via mlflow -statsd==3.3.0 - # via flytekit -tabulate==0.9.0 - # via databricks-cli -tenacity==8.2.3 - # via plotly -tensorboard==2.16.2 - # via tensorflow -tensorboard-data-server==0.7.2 - # via tensorboard -tensorflow==2.16.1 - # via -r requirements.in -tensorflow-io-gcs-filesystem==0.36.0 - # via tensorflow -termcolor==2.4.0 - # via tensorflow -text-unidecode==1.3 - # via python-slugify -types-python-dateutil==2.9.0.20240316 - # via arrow -typing-extensions==4.11.0 - # via - # alembic - # azure-core - # azure-storage-blob - # flytekit - # mashumaro - # optree - # rich-click - # tensorflow - # typing-inspect -typing-inspect==0.9.0 - # via dataclasses-json -urllib3==1.26.18 - # via - # botocore - # databricks-cli - # docker - # flytekit - # requests -websocket-client==1.7.0 - # via docker -werkzeug==3.0.2 - # via - # flask - # tensorboard -wheel==0.43.0 - # via astunparse -wrapt==1.16.0 - # via - # aiobotocore - # tensorflow -yarl==1.9.4 - # via aiohttp -zipp==3.18.1 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools From 5c7946676f5d574fbc260b5a60e0646ff70372b1 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 18:54:49 +0800 Subject: [PATCH 35/47] test Signed-off-by: Kevin Su --- .github/actions/clear-action-cache/action.yml | 11 +++++++++++ .github/workflows/checks_new.yml | 2 ++ 2 files changed, 13 insertions(+) create mode 100644 .github/actions/clear-action-cache/action.yml diff --git a/.github/actions/clear-action-cache/action.yml b/.github/actions/clear-action-cache/action.yml new file mode 100644 index 000000000..a29347b61 --- /dev/null +++ b/.github/actions/clear-action-cache/action.yml @@ -0,0 +1,11 @@ +name: 'Clear action cache' +description: 'As suggested by GitHub to prevent low disk space: https://github.com/actions/runner-images/issues/2840#issuecomment-790492173' +runs: + using: 'composite' + steps: + - shell: bash + run: | + rm -rf /usr/share/dotnet + rm -rf /opt/ghc + rm -rf "/usr/local/share/boost" + rm -rf "$AGENT_TOOLSDIRECTORY" diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 5164fa4e8..0b8da49eb 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -45,6 +45,8 @@ jobs: example: "${{ fromJson(needs.list_examples.outputs.packages) }}" steps: - uses: actions/checkout@v4 + - name: 'Clear action cache' + uses: ./.github/actions/clear-action-cache - name: Set up Python uses: actions/setup-python@v4 with: From 489d145dbc49a4436bdecc26bec7251bc677f701 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 19:07:56 +0800 Subject: [PATCH 36/47] test Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 0b8da49eb..9ad9c15f6 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -26,7 +26,7 @@ jobs: run: | echo "PACKAGES=$(find examples -maxdepth 1 -type d -exec basename '{}' \; \ | grep -v -e 'testing' -e 'examples' \ - | grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' -e 'feast_integration' -e 'modin_plugin' -e 'sagemaker_inference_agent' \ + | grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' -e 'feast_integration' -e 'modin_plugin' -e 'sagemaker_inference_agent' -e 'kfpytorch_plugin' \ | sort \ | jq --raw-input . \ | jq --slurp . \ From 67d9b617c9f0ebc74acd55e469988bb22c91114a Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 19:34:19 +0800 Subject: [PATCH 37/47] test Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 2 +- examples/sagemaker_inference_agent/requirements.in | 6 +++--- scripts/serialize-example.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 9ad9c15f6..e0833fdb4 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -86,7 +86,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v2 with: - name: snacks-${{ matrix.example }} + name: snacks-examples-${{ matrix.example }} path: examples/${{ matrix.example }}/**/*.pb # Download all artifacts generated from the previous job. Startup a sandbox cluster then register all of them. diff --git a/examples/sagemaker_inference_agent/requirements.in b/examples/sagemaker_inference_agent/requirements.in index 1160374e9..975e20259 100644 --- a/examples/sagemaker_inference_agent/requirements.in +++ b/examples/sagemaker_inference_agent/requirements.in @@ -1,7 +1,7 @@ -flytekitplugins-awssagemaker==1.12.0b2 +flytekitplugins-awssagemaker xgboost fastapi uvicorn scikit-learn -flytekit==1.11.0 -flyteidl==1.11.1b0 +flytekit +flyteidl diff --git a/scripts/serialize-example.sh b/scripts/serialize-example.sh index 5898a3ae6..57965cf64 100755 --- a/scripts/serialize-example.sh +++ b/scripts/serialize-example.sh @@ -17,7 +17,7 @@ serialize() { --image "$2" \ --image mindmeld="ghcr.io/flyteorg/flytecookbook:core-latest" \ --image borebuster="ghcr.io/flyteorg/flytekit:py3.9-latest" \ - --output flyte-package.tgz \ + --output /root/flyte-package.tgz \ --force } From 28d4fa52280832e738cbce2259e7bffb963a91bb Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 19:57:08 +0800 Subject: [PATCH 38/47] test Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index e0833fdb4..bdcf0bb2d 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -26,7 +26,7 @@ jobs: run: | echo "PACKAGES=$(find examples -maxdepth 1 -type d -exec basename '{}' \; \ | grep -v -e 'testing' -e 'examples' \ - | grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' -e 'feast_integration' -e 'modin_plugin' -e 'sagemaker_inference_agent' -e 'kfpytorch_plugin' \ + | grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' -e 'feast_integration' -e 'modin_plugin' -e 'sagemaker_inference_agent' -e 'kfpytorch_plugin' -e 'extending' \ | sort \ | jq --raw-input . \ | jq --slurp . \ From d9fa22d4fe8d554aaaf7c4e39b302254ec4a0860 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 21:32:35 +0800 Subject: [PATCH 39/47] test Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 3 ++- .../chatgpt_agent/chatgpt_agent_example_usage.py | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index bdcf0bb2d..16bdcb113 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -24,9 +24,10 @@ jobs: - name: "Provide the list" id: create-example-list run: | + # TODO: Register and update the examples below. (onnx_plugin, feast_integration, etc) echo "PACKAGES=$(find examples -maxdepth 1 -type d -exec basename '{}' \; \ | grep -v -e 'testing' -e 'examples' \ - | grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' -e 'feast_integration' -e 'modin_plugin' -e 'sagemaker_inference_agent' -e 'kfpytorch_plugin' -e 'extending' \ + | grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' -e 'feast_integration' -e 'modin_plugin' -e 'sagemaker_inference_agent' -e 'kfpytorch_plugin'' \ | sort \ | jq --raw-input . \ | jq --slurp . \ diff --git a/examples/chatgpt_agent/chatgpt_agent/chatgpt_agent_example_usage.py b/examples/chatgpt_agent/chatgpt_agent/chatgpt_agent_example_usage.py index e778a9f64..fc8974307 100644 --- a/examples/chatgpt_agent/chatgpt_agent/chatgpt_agent_example_usage.py +++ b/examples/chatgpt_agent/chatgpt_agent/chatgpt_agent_example_usage.py @@ -130,14 +130,14 @@ def post_message_on_slack(message: str): @workflow -def wf(owner: str = "flyteorg", repo: str = "flyte", channel: str = "demo"): +def slack_wf(owner: str = "flyteorg", repo: str = "flyte", channel: str = "demo"): message = get_github_latest_release(owner=owner, repo=repo) message = chatgpt_job(message=message) post_message_on_slack(message=message) if __name__ == "__main__": - wf() + slack_wf() # %% [markdown] @@ -183,7 +183,7 @@ def get_latest_video_transcript_chunks(channel_url: str) -> List[str]: @workflow -def wf(channel_url: str): +def video_wf(channel_url: str): chunks = get_latest_video_transcript_chunks(channel_url=channel_url) dynamic_subwf(channel_url=channel_url, chunks=chunks) @@ -268,7 +268,7 @@ def dynamic_subwf(channel_url: str, chunks: List[str]): if __name__ == "__main__": - wf(channel_url="https://www.youtube.com/@flyteorg") + video_wf(channel_url="https://www.youtube.com/@flyteorg") # %% [markdown] # ### Summarize the latest MLOps trend from Medium to Twitter @@ -353,11 +353,11 @@ def tweet(text: str): @workflow -def wf(url: str = "https://medium.com/tag/flyte"): +def tweet_wf(url: str = "https://medium.com/tag/flyte"): message = get_weekly_articles_title(url=url) message = chatgpt_job(message=message) tweet(text=message) if __name__ == "__main__": - wf() + tweet_wf() From 0b35a973980b65e73c44dbdc3635aa370002cffe Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 21:33:47 +0800 Subject: [PATCH 40/47] nit Signed-off-by: Kevin Su --- .github/workflows/checks_new.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml index 16bdcb113..d902e5cb6 100644 --- a/.github/workflows/checks_new.yml +++ b/.github/workflows/checks_new.yml @@ -27,7 +27,7 @@ jobs: # TODO: Register and update the examples below. (onnx_plugin, feast_integration, etc) echo "PACKAGES=$(find examples -maxdepth 1 -type d -exec basename '{}' \; \ | grep -v -e 'testing' -e 'examples' \ - | grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' -e 'feast_integration' -e 'modin_plugin' -e 'sagemaker_inference_agent' -e 'kfpytorch_plugin'' \ + | grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' -e 'feast_integration' -e 'modin_plugin' -e 'sagemaker_inference_agent' -e 'kfpytorch_plugin' \ | sort \ | jq --raw-input . \ | jq --slurp . \ From 2d1a623d05cae2c76e1a5d06a8e0664cb905ecff Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 21:48:17 +0800 Subject: [PATCH 41/47] tet Signed-off-by: Kevin Su --- .github/workflows/checks.yml | 144 +++++++++++++++++++++---------- .github/workflows/checks_new.yml | 121 -------------------------- 2 files changed, 100 insertions(+), 165 deletions(-) delete mode 100644 .github/workflows/checks_new.yml diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 13b03ed41..c9cc6bdfd 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -6,9 +6,9 @@ on: - master paths-ignore: - "docs/**" -# pull_request: -# paths-ignore: -# - "docs/**" + pull_request: + paths-ignore: + - "docs/**" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.11"] + python-version: [ "3.11" ] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -41,7 +41,6 @@ jobs: run: python -m pip list - name: Lint run: pre-commit run --all --show-diff-on-failure - # This is the build system for the new example directory structure list_examples: runs-on: ubuntu-latest @@ -50,59 +49,114 @@ jobs: - uses: actions/checkout@v2 - name: "Provide the list" id: create-example-list - run: echo "PACKAGES=$(find examples -mindepth 1 -maxdepth 2 -type f -name Dockerfile -exec dirname '{}' \; | sort | jq --raw-input . | jq --slurp . | jq -c .)" >> "$GITHUB_OUTPUT" + run: | + # TODO: Register and update the examples below. (onnx_plugin, feast_integration, etc) + echo "PACKAGES=$(find examples -maxdepth 1 -type d -exec basename '{}' \; \ + | grep -v -e 'testing' -e 'examples' \ + | grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' -e 'feast_integration' -e 'modin_plugin' -e 'sagemaker_inference_agent' -e 'kfpytorch_plugin' \ + | sort \ + | jq --raw-input . \ + | jq --slurp . \ + | jq -c .)" >> "$GITHUB_OUTPUT" outputs: packages: "${{ steps.create-example-list.outputs.PACKAGES }}" - - trigger_serialize_register_examples: - name: Serialize & Register Flytesnacks workflow - needs: [list_examples] - uses: ./.github/workflows/serialize_example.yml - with: - packages: ${{ needs.list_examples.outputs.PACKAGES }} - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - push_example_image_to_github: - name: Build & Push Example Image to GHCR - runs-on: ubuntu-latest + serialize: needs: [list_examples] + runs-on: "ubuntu-latest" + env: + FLYTE_SDK_RICH_TRACEBACKS: "0" + PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python strategy: + fail-fast: false matrix: - directory: "${{ fromJson(needs.list_examples.outputs.packages) }}" + example: "${{ fromJson(needs.list_examples.outputs.packages) }}" + steps: + - uses: actions/checkout@v4 + - name: 'Clear action cache' + uses: ./.github/actions/clear-action-cache + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Cache pip + uses: actions/cache@v3 + with: + # This path is specific to Ubuntu + path: ~/.cache/pip + # Look to see if there is a cache hit for the corresponding requirements files + key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }} + - name: Install dependencies + working-directory: examples/${{ matrix.example }} + run: | + pip install uv + uv venv + source .venv/bin/activate + uv pip install flytekit flytekitplugins-envd + if [ -f requirements.in ]; then uv pip install -r requirements.in; fi + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ secrets.FLYTE_BOT_USERNAME }} + password: ${{ secrets.FLYTE_BOT_PAT }} + - name: Pyflyte package + working-directory: examples/${{ matrix.example }} + run: | + source .venv/bin/activate + pyflyte \ + --pkgs ${{ matrix.example }} package \ + --image mindmeld="ghcr.io/flyteorg/flytecookbook:core-latest" \ + --image borebuster="ghcr.io/flyteorg/flytekit:py3.9-latest" \ + --output flyte-package.tgz \ + --force + tar -xvf flyte-package.tgz + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: snacks-examples-${{ matrix.example }} + path: examples/${{ matrix.example }}/**/*.pb + + # Download all artifacts generated from the previous job. Startup a sandbox cluster then register all of them. + register: + name: Register example to sandbox + runs-on: ubuntu-latest + needs: [ "serialize" ] steps: - uses: actions/checkout@v2 with: fetch-depth: "0" - - name: "Set example name" - id: example_id - run: echo "EXAMPLE_NAME=$(basename -- ${{ matrix.directory }})" >> "$GITHUB_OUTPUT" - - name: Build & Push Docker Image to Github Registry - uses: whoan/docker-build-with-cache-action@v5 - with: - # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - image_name: ${{ github.repository_owner }}/flytecookbook - image_tag: ${{ steps.example_id.outputs.EXAMPLE_NAME }}-latest,${{ steps.example_id.outputs.EXAMPLE_NAME }}-${{ github.sha }} - registry: ghcr.io - push_git_tag: ${{ github.event_name != 'pull_request' }} - push_image_and_stages: ${{ github.event_name != 'pull_request' }} - build_extra_args: "--compress=true --build-arg=tag=ghcr.io/${{ github.repository_owner }}/flytecookbook:${{ steps.example_id.outputs.id }}-${{ github.sha }}" - context: ${{ matrix.directory }} - dockerfile: Dockerfile + - uses: unionai/flytectl-setup-action@v0.0.1 + - name: setup download artifact dir + run: | + mkdir download-artifact + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + path: ./download-artifact/ + - name: setup sandbox + run: | + flytectl demo start + flytectl config init + - name: Register examples + uses: unionai/flyte-register-action@v0.0.2 + with: + flytesnacks: false + proto: ./download-artifact/**/* + project: flytesnacks + version: "latest" + domain: development bump_version: name: Bump Version if: ${{ github.event_name != 'pull_request' }} - needs: [trigger_serialize_register_examples, push_example_image_to_github] # Only to ensure it can successfully build + needs: [ trigger_serialize_register_examples, push_example_image_to_github ] # Only to ensure it can successfully build uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master secrets: FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} prerelease: name: Create Prerelease - needs: [bump_version] + needs: [ bump_version ] runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -126,10 +180,10 @@ jobs: release_workflow: name: Publish artifacts to github release runs-on: ubuntu-latest - needs: [prerelease] + needs: [ prerelease ] strategy: matrix: - python-version: ["3.11"] + python-version: [ "3.11" ] steps: - uses: actions/checkout@v2 with: @@ -164,7 +218,7 @@ jobs: make_release: name: Mark github pre-release as Release - needs: [release_workflow] + needs: [ release_workflow ] runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -216,8 +270,10 @@ jobs: flytectl demo start --imagePullPolicy Never - name: Install Python dependencies run: | - python -m pip install --upgrade pip - pip install flytekit flytekitplugins-deck-standard torch + pip install uv + uv venv + source .venv/bin/activate + uv pip install --upgrade pip flytekit flytekitplugins-deck-standard torch pip freeze - name: Checkout flytesnacks uses: actions/checkout@v3 diff --git a/.github/workflows/checks_new.yml b/.github/workflows/checks_new.yml deleted file mode 100644 index d902e5cb6..000000000 --- a/.github/workflows/checks_new.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: Master - -on: - push: - branches: - - master - paths-ignore: - - "docs/**" - pull_request: - paths-ignore: - - "docs/**" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - # This is the build system for the new example directory structure - list_examples: - runs-on: ubuntu-latest - name: "Create a list of example packages" - steps: - - uses: actions/checkout@v2 - - name: "Provide the list" - id: create-example-list - run: | - # TODO: Register and update the examples below. (onnx_plugin, feast_integration, etc) - echo "PACKAGES=$(find examples -maxdepth 1 -type d -exec basename '{}' \; \ - | grep -v -e 'testing' -e 'examples' \ - | grep -v -e 'airflow_plugin' -e 'forecasting_sales' -e 'onnx_plugin' -e 'feast_integration' -e 'modin_plugin' -e 'sagemaker_inference_agent' -e 'kfpytorch_plugin' \ - | sort \ - | jq --raw-input . \ - | jq --slurp . \ - | jq -c .)" >> "$GITHUB_OUTPUT" - outputs: - packages: "${{ steps.create-example-list.outputs.PACKAGES }}" - serialize: - needs: [list_examples] - runs-on: "ubuntu-latest" - env: - FLYTE_SDK_RICH_TRACEBACKS: "0" - PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python - strategy: - fail-fast: false - matrix: - example: "${{ fromJson(needs.list_examples.outputs.packages) }}" - steps: - - uses: actions/checkout@v4 - - name: 'Clear action cache' - uses: ./.github/actions/clear-action-cache - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - name: Cache pip - uses: actions/cache@v3 - with: - # This path is specific to Ubuntu - path: ~/.cache/pip - # Look to see if there is a cache hit for the corresponding requirements files - key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }} - - name: Install dependencies - working-directory: examples/${{ matrix.example }} - run: | - pip install uv - uv venv - source .venv/bin/activate - uv pip install flytekit flytekitplugins-envd - if [ -f requirements.in ]; then uv pip install -r requirements.in; fi - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ secrets.FLYTE_BOT_USERNAME }} - password: ${{ secrets.FLYTE_BOT_PAT }} - - name: Pyflyte package - working-directory: examples/${{ matrix.example }} - run: | - source .venv/bin/activate - pyflyte \ - --pkgs ${{ matrix.example }} package \ - --image mindmeld="ghcr.io/flyteorg/flytecookbook:core-latest" \ - --image borebuster="ghcr.io/flyteorg/flytekit:py3.9-latest" \ - --output flyte-package.tgz \ - --force - tar -xvf flyte-package.tgz - - name: Upload artifacts - uses: actions/upload-artifact@v2 - with: - name: snacks-examples-${{ matrix.example }} - path: examples/${{ matrix.example }}/**/*.pb - - # Download all artifacts generated from the previous job. Startup a sandbox cluster then register all of them. - register: - name: Register example to sandbox - runs-on: ubuntu-latest - needs: [ "serialize" ] - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - uses: unionai/flytectl-setup-action@v0.0.1 - - name: setup download artifact dir - run: | - mkdir download-artifact - - name: Download artifacts - uses: actions/download-artifact@v2 - with: - path: ./download-artifact/ - - name: setup sandbox - run: | - flytectl demo start - flytectl config init - - name: Register examples - uses: unionai/flyte-register-action@v0.0.2 - with: - flytesnacks: false - proto: ./download-artifact/**/* - project: flytesnacks - version: "latest" - domain: development \ No newline at end of file From d2f70a61555cb2f0f362fa02dc4a1bfb37271a52 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 21:51:52 +0800 Subject: [PATCH 42/47] test Signed-off-by: Kevin Su --- .github/workflows/monodocs_build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/monodocs_build.yml b/.github/workflows/monodocs_build.yml index d3bdfeecb..833910e76 100644 --- a/.github/workflows/monodocs_build.yml +++ b/.github/workflows/monodocs_build.yml @@ -8,9 +8,9 @@ on: push: branches: - master -# pull_request: -# branches: -# - master + pull_request: + branches: + - master jobs: docs: name: Docs Build From 40d6ad2b9791aa67ce9e280277a5fd4281d8f960 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 22:04:17 +0800 Subject: [PATCH 43/47] update decks example Signed-off-by: Kevin Su --- .../development_lifecycle/development_lifecycle/decks.py | 4 ++-- examples/development_lifecycle/requirements.in | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/development_lifecycle/development_lifecycle/decks.py b/examples/development_lifecycle/development_lifecycle/decks.py index 84d93e090..9a69ee7be 100644 --- a/examples/development_lifecycle/development_lifecycle/decks.py +++ b/examples/development_lifecycle/development_lifecycle/decks.py @@ -3,7 +3,7 @@ from flytekitplugins.deck.renderer import MarkdownRenderer from sklearn.decomposition import PCA -# Ceate a new deck named `pca` and render Markdown content along with a +# Create a new deck named `pca` and render Markdown content along with a # PCA (https://en.wikipedia.org/wiki/Principal_component_analysis) plot. # Start by initializing an `ImageSpec`` object # to encompass all the necessary dependencies. @@ -12,7 +12,7 @@ # For more information, see # https://docs.flyte.org/en/latest/user_guide/customizing_dependencies/imagespec.html#image-spec-example -custom_image = ImageSpec(name="flyte-decks-example", packages=["plotly"], registry="ghcr.io/flyteorg") +custom_image = ImageSpec(packages=["plotly"], registry="ghcr.io/flyteorg") if custom_image.is_container(): import plotly diff --git a/examples/development_lifecycle/requirements.in b/examples/development_lifecycle/requirements.in index 9bca62dfe..c9db52868 100644 --- a/examples/development_lifecycle/requirements.in +++ b/examples/development_lifecycle/requirements.in @@ -1,5 +1,5 @@ -flytekit>=1.10.3b4 -flytekitplugins-deck-standard>=1.10.3b4 -flytekitplugins-envd>=1.10.3b4 +flytekit +flytekitplugins-deck-standard +flytekitplugins-envd plotly scikit-learn From 36aacfb9921d16d4201bdc5f393d8f925328b056 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 22:28:34 +0800 Subject: [PATCH 44/47] nit Signed-off-by: Kevin Su --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index c9cc6bdfd..ad98c6894 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -149,7 +149,7 @@ jobs: bump_version: name: Bump Version if: ${{ github.event_name != 'pull_request' }} - needs: [ trigger_serialize_register_examples, push_example_image_to_github ] # Only to ensure it can successfully build + needs: [ serialize ] # Only to ensure it can successfully build uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master secrets: FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} From 4bd8fb9badbbf9fc72173c805867fdbbe54a2a70 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 22:42:49 +0800 Subject: [PATCH 45/47] lint Signed-off-by: Kevin Su --- examples/chatgpt_agent/requirements.in | 2 +- examples/modin_plugin/requirements.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/chatgpt_agent/requirements.in b/examples/chatgpt_agent/requirements.in index 28e9bf61f..e0457edf1 100644 --- a/examples/chatgpt_agent/requirements.in +++ b/examples/chatgpt_agent/requirements.in @@ -1 +1 @@ -flytekitplugins-openai \ No newline at end of file +flytekitplugins-openai diff --git a/examples/modin_plugin/requirements.in b/examples/modin_plugin/requirements.in index 065912b20..06adef7f5 100644 --- a/examples/modin_plugin/requirements.in +++ b/examples/modin_plugin/requirements.in @@ -6,4 +6,4 @@ flytekitplugins-modin scikit-learn modin ray -pandas \ No newline at end of file +pandas From 337fdf070640341c58717fe0ebf3a013b9474fc9 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 23:14:50 +0800 Subject: [PATCH 46/47] nit Signed-off-by: Kevin Su --- .github/workflows/checks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ad98c6894..be66b68a2 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -285,6 +285,7 @@ jobs: python flyte_tests_validate.py - name: Register specific tests run: | + source .venv/bin/activate while read -r line; do pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ From 5ff5cda83059e9f18ec049d0bc750450656e0459 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 23 Apr 2024 23:45:08 +0800 Subject: [PATCH 47/47] nit Signed-off-by: Kevin Su --- .github/workflows/checks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index be66b68a2..9cbe85f95 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -282,6 +282,7 @@ jobs: path: flytesnacks - name: Verify existence of the tests run: | + source .venv/bin/activate python flyte_tests_validate.py - name: Register specific tests run: |