Skip to content

Commit

Permalink
Merge branch 'master' into #5319
Browse files Browse the repository at this point in the history
  • Loading branch information
Mecoli1219 authored Sep 27, 2024
2 parents aa1760c + 98d722f commit df0f898
Show file tree
Hide file tree
Showing 298 changed files with 14,789 additions and 2,714 deletions.
9 changes: 4 additions & 5 deletions .github/workflows/monodocs_build.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name: Monodocs Build

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
cancel-in-progress: true

on:
Expand All @@ -18,8 +18,8 @@ jobs:
steps:
- name: Fetch flytekit code
uses: actions/checkout@v4
with:
path: "${{ github.workspace }}/flytekit"
- name: 'Clear action cache'
uses: ./.github/actions/clear-action-cache
- name: Fetch flyte code
uses: actions/checkout@v4
with:
Expand All @@ -41,7 +41,6 @@ jobs:
export SETUPTOOLS_SCM_PRETEND_VERSION="2.0.0"
pip install -e ./flyteidl
- shell: bash -el {0}
working-directory: ${{ github.workspace }}/flytekit
run: |
conda activate monodocs-env
pip install -e .
Expand All @@ -54,7 +53,7 @@ jobs:
working-directory: ${{ github.workspace }}/flyte
shell: bash -el {0}
env:
FLYTEKIT_LOCAL_PATH: ${{ github.workspace }}/flytekit
FLYTEKIT_LOCAL_PATH: ${{ github.workspace }}
run: |
conda activate monodocs-env
make -C docs clean html SPHINXOPTS="-W -vvv"
39 changes: 25 additions & 14 deletions .github/workflows/pythonbuild.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ env:
FLYTE_SDK_LOGGING_LEVEL: 10 # debug

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
cancel-in-progress: true

jobs:
Expand All @@ -28,7 +28,7 @@ jobs:
if [[ ${{ github.event_name }} == "schedule" ]]; then
echo "python_versions=[\"3.8\",\"3.9\",\"3.10\",\"3.11\",\"3.12\"]" >> $GITHUB_ENV
else
echo "python_versions=[\"3.12\"]" >> $GITHUB_ENV
echo "python_versions=[\"3.9\", \"3.12\"]" >> $GITHUB_ENV
fi
build:
Expand All @@ -42,7 +42,7 @@ jobs:
python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
steps:
- uses: actions/checkout@v4
- name: 'Clear action cache'
- name: "Clear action cache"
uses: ./.github/actions/clear-action-cache
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
Expand All @@ -59,7 +59,7 @@ jobs:
run: |
pip install uv
make setup-global-uv
uv pip uninstall --system pandas
uv pip uninstall --system pandas pyarrow
uv pip freeze
- name: Test with coverage
run: |
Expand All @@ -81,7 +81,7 @@ jobs:
python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
steps:
- uses: actions/checkout@v4
- name: 'Clear action cache'
- name: "Clear action cache"
uses: ./.github/actions/clear-action-cache
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
Expand All @@ -98,7 +98,7 @@ jobs:
run: |
pip install uv
make setup-global-uv
uv pip uninstall --system pandas
uv pip uninstall --system pandas pyarrow
uv pip freeze
- name: Run extras unit tests with coverage
# Skip this step if running on python 3.12 due to https://github.com/tensorflow/tensorflow/issues/62003
Expand Down Expand Up @@ -128,10 +128,12 @@ jobs:
pandas: "pandas<2.0.0"
- numpy: "numpy<2.0.0"
pandas: "pandas>=2.0.0"
- numpy: "numpy>=2.0.0"
python-version: "3.8"

steps:
- uses: actions/checkout@v4
- name: 'Clear action cache'
- name: "Clear action cache"
uses: ./.github/actions/clear-action-cache
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
Expand Down Expand Up @@ -242,12 +244,16 @@ jobs:
matrix:
os: [ubuntu-latest]
python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
makefile-cmd:
[integration_test_codecov, integration_test_lftransfers_codecov]
steps:
# As described in https://github.com/pypa/setuptools_scm/issues/414, SCM needs git history
# and tags to work.
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: "Clear action cache"
uses: ./.github/actions/clear-action-cache # sandbox has disk pressure, so we need to clear the cache to get more disk space.
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
Expand Down Expand Up @@ -293,7 +299,7 @@ jobs:
FLYTEKIT_CI: 1
PYTEST_OPTS: -n2
run: |
make integration_test_codecov
make ${{ matrix.makefile-cmd }}
- name: Codecov
uses: codecov/[email protected]
with:
Expand All @@ -316,6 +322,7 @@ jobs:
- flytekit-aws-batch
- flytekit-aws-sagemaker
- flytekit-bigquery
- flytekit-comet-ml
- flytekit-dask
- flytekit-data-fsspec
- flytekit-dbt
Expand All @@ -329,18 +336,21 @@ jobs:
- flytekit-hive
- flytekit-huggingface
- flytekit-identity-aware-proxy
- flytekit-inference
- flytekit-k8s-pod
- flytekit-kf-mpi
- flytekit-kf-pytorch
- flytekit-kf-tensorflow
- flytekit-mlflow
- flytekit-mmcloud
- flytekit-modin
- flytekit-neptune
- flytekit-onnx-pytorch
- flytekit-onnx-scikitlearn
# onnx-tensorflow needs a version of tensorflow that does not work with protobuf>4.
# The issue is being tracked on the tensorflow side in https://github.com/tensorflow/tensorflow/issues/53234#issuecomment-1330111693
# flytekit-onnx-tensorflow
- flytekit-omegaconf
- flytekit-openai
- flytekit-pandera
- flytekit-papermill
Expand All @@ -353,6 +363,10 @@ jobs:
- flytekit-vaex
- flytekit-whylogs
exclude:
- python-version: 3.8
plugin-names: "flytekit-aws-sagemaker"
- python-version: 3.9
plugin-names: "flytekit-aws-sagemaker"
# flytekit-modin depends on ray which does not have a 3.11 wheel yet.
# Issue tracked in https://github.com/ray-project/ray/issues/27881
- python-version: 3.11
Expand Down Expand Up @@ -382,9 +396,6 @@ jobs:
# apache-beam, one of flytekit-airflow dependencies, does not support python 3.12: https://github.com/apache/beam/issues/29149
- python-version: 3.12
plugin-names: "flytekit-airflow"
# ydata-profiling, a dependency of flytekit-deck-standard, does not support python 3.12: https://github.com/ydataai/ydata-profiling/issues/1510
- python-version: 3.12
plugin-names: "flytekit-deck-standard"
# Tensorflow is a dependency of flytekit-mlflow tests and that is not supported yet: https://github.com/tensorflow/tensorflow/issues/62003
- python-version: 3.12
plugin-names: "flytekit-mlflow"
Expand All @@ -402,7 +413,7 @@ jobs:
plugin-names: "flytekit-kf-pytorch"
steps:
- uses: actions/checkout@v4
- name: 'Clear action cache'
- name: "Clear action cache"
uses: ./.github/actions/clear-action-cache
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
Expand All @@ -425,9 +436,9 @@ jobs:
uv pip install --system .
if [ -f dev-requirements.in ]; then uv pip install --system -r dev-requirements.in; fi
# TODO: move to protobuf>=5. Github issue: https://github.com/flyteorg/flyte/issues/5448
uv pip install --system -U $GITHUB_WORKSPACE "protobuf<5"
uv pip install --system -U $GITHUB_WORKSPACE "protobuf<5" "git+https://github.com/flyteorg/flyte.git@master#subdirectory=flyteidl"
# TODO: remove this when numpy v2 in onnx has been resolved
if [[ ${{ matrix.plugin-names }} == *"onnx"* || ${{ matrix.plugin-names }} == "flytekit-sqlalchemy" ]]; then
if [[ ${{ matrix.plugin-names }} == *"onnx"* || ${{ matrix.plugin-names }} == "flytekit-sqlalchemy" || ${{ matrix.plugin-names }} == "flytekit-pandera" ]]; then
uv pip install --system "numpy<2.0.0"
fi
uv pip freeze
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ docs/source/_tags/
.hypothesis
.npm
/**/target
coverage.xml

# Version file is auto-generated by setuptools_scm
flytekit/_version.py
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ repos:
hooks:
- id: check_pdb_hook
- repo: https://github.com/codespell-project/codespell
rev: v2.2.6
rev: v2.3.0
hooks:
- id: codespell
additional_dependencies:
Expand Down
3 changes: 2 additions & 1 deletion CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
# These owners will be the default owners for everything in
# the repo. Unless a later match takes precedence.
* @wild-endeavor @kumare3 @eapolinario @pingsutw @cosmicBboy @samhita-alla
* @wild-endeavor @kumare3 @eapolinario @pingsutw @cosmicBboy @samhita-alla @thomasjpfan @future-outlier
plugins/flytekit-kf-pytorch @fg91 @wild-endeavor @kumare3 @eapolinario @pingsutw @cosmicBboy @samhita-alla @thomasjpfan @future-outlier
13 changes: 7 additions & 6 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
ARG PYTHON_VERSION
ARG PYTHON_VERSION=3.12
FROM python:${PYTHON_VERSION}-slim-bookworm

MAINTAINER Flyte Team <[email protected]>
LABEL org.opencontainers.image.authors="Flyte Team <[email protected]>"
LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytekit

WORKDIR /root
ENV PYTHONPATH /root
ENV FLYTE_SDK_RICH_TRACEBACKS 0
ENV PYTHONPATH=/root
ENV FLYTE_SDK_RICH_TRACEBACKS=0

ARG VERSION
ARG DOCKER_IMAGE
Expand All @@ -23,15 +23,16 @@ ARG DOCKER_IMAGE
RUN apt-get update && apt-get install build-essential -y \
&& pip install uv \
&& uv pip install --system --no-cache-dir -U flytekit==$VERSION \
flytekitplugins-deck-standard==$VERSION \
kubernetes \
&& apt-get clean autoclean \
&& apt-get autoremove --yes \
&& rm -rf /var/lib/{apt,dpkg,cache,log}/ \
&& rm -rf /root/.cache/pip \
&& useradd -u 1000 flytekit \
&& chown flytekit: /root \
&& chown flytekit: /home \
&& :

USER flytekit

ENV FLYTE_INTERNAL_IMAGE "$DOCKER_IMAGE"
ENV FLYTE_INTERNAL_IMAGE="$DOCKER_IMAGE"
8 changes: 4 additions & 4 deletions Dockerfile.agent
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
FROM python:3.10-slim-bookworm as agent-slim
FROM python:3.10-slim-bookworm AS agent-slim

MAINTAINER Flyte Team <[email protected]>
LABEL org.opencontainers.image.authors="Flyte Team <[email protected]>"
LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytekit

ARG VERSION
Expand All @@ -19,9 +19,9 @@ RUN pip install --no-cache-dir -U flytekit==$VERSION \
&& rm -rf /var/lib/{apt,dpkg,cache,log}/ \
&& :

CMD pyflyte serve agent --port 8000
CMD ["pyflyte", "serve", "agent", "--port", "8000"]

FROM agent-slim as agent-all
FROM agent-slim AS agent-all
ARG VERSION

RUN pip install --no-cache-dir -U \
Expand Down
17 changes: 11 additions & 6 deletions Dockerfile.dev
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,17 @@
# From your test user code
# $ pyflyte run --image localhost:30000/flytekittest:someversion

ARG PYTHON_VERSION
ARG PYTHON_VERSION=3.12
FROM python:${PYTHON_VERSION}-slim-bookworm

MAINTAINER Flyte Team <[email protected]>
LABEL org.opencontainers.image.authors="Flyte Team <[email protected]>"
LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytekit

WORKDIR /root
ENV FLYTE_SDK_RICH_TRACEBACKS 0
ENV FLYTE_SDK_RICH_TRACEBACKS=0

# Flytekit version of flytekit to be installed in the image
ARG PSEUDO_VERSION
ARG PSEUDO_VERSION=1.13.3


# Note: Pod tasks should be exposed in the default image
Expand All @@ -38,10 +38,15 @@ RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTEKIT=$PSEUDO_VERSION \
uv pip install --system --no-cache-dir -U \
"git+https://github.com/flyteorg/flyte.git@master#subdirectory=flyteidl" \
-e /flytekit \
-e /flytekit/plugins/flytekit-k8s-pod \
-e /flytekit/plugins/flytekit-deck-standard \
-e /flytekit/plugins/flytekit-flyteinteractive \
markdown \
pandas \
pillow \
plotly \
pygments \
scikit-learn \
ydata-profiling \
&& apt-get clean autoclean \
&& apt-get autoremove --yes \
&& rm -rf /var/lib/{apt,dpkg,cache,log}/ \
Expand All @@ -51,7 +56,7 @@ RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTEKIT=$PSEUDO_VERSION \
&& :


ENV PYTHONPATH "/flytekit:/flytekit/plugins/flytekit-k8s-pod:/flytekit/plugins/flytekit-deck-standard:"
ENV PYTHONPATH="/flytekit:/flytekit/tests/flytekit/integration/remote"

# Switch to the 'flytekit' user for better security.
USER flytekit
11 changes: 10 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,15 @@ integration_test_codecov:

.PHONY: integration_test
integration_test:
$(PYTEST_AND_OPTS) tests/flytekit/integration ${CODECOV_OPTS}
$(PYTEST_AND_OPTS) tests/flytekit/integration ${CODECOV_OPTS} -m "not lftransfers"

.PHONY: integration_test_lftransfers_codecov
integration_test_lftransfers_codecov:
$(MAKE) CODECOV_OPTS="--cov=./ --cov-report=xml --cov-append" integration_test_lftransfers

.PHONY: integration_test_lftransfers
integration_test_lftransfers:
$(PYTEST) tests/flytekit/integration ${CODECOV_OPTS} -m "lftransfers"

doc-requirements.txt: export CUSTOM_COMPILE_COMMAND := make doc-requirements.txt
doc-requirements.txt: doc-requirements.in install-piptools
Expand All @@ -119,5 +127,6 @@ build-dev: export PLATFORM ?= linux/arm64
build-dev: export REGISTRY ?= localhost:30000
build-dev: export PYTHON_VERSION ?= 3.12
build-dev: export PSEUDO_VERSION ?= $(shell python -m setuptools_scm)
build-dev: export TAG ?= dev
build-dev:
docker build --platform ${PLATFORM} --push . -f Dockerfile.dev -t ${REGISTRY}/flytekit:${TAG} --build-arg PYTHON_VERSION=${PYTHON_VERSION} --build-arg PSEUDO_VERSION=${PSEUDO_VERSION}
5 changes: 3 additions & 2 deletions dev-requirements.in
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,14 @@ pre-commit
codespell
google-cloud-bigquery
google-cloud-bigquery-storage
snowflake-connector-python
IPython
keyrings.alt
setuptools_scm
pytest-icdiff
jinja2

# Tensorflow is not available for python 3.12 yet: https://github.com/tensorflow/tensorflow/issues/62003
tensorflow; python_version<'3.12'
tensorflow<=2.15.1; python_version<'3.12'
# Newer versions of torch bring in nvidia dependencies that are not present in windows, so
# we put this constraint while we do not have per-environment requirements files
torch<=1.12.1; python_version<'3.11'
Expand Down Expand Up @@ -51,6 +51,7 @@ autoflake
pillow
numpy
pandas
pyarrow
scikit-learn
types-requests
prometheus-client
Expand Down
Loading

0 comments on commit df0f898

Please sign in to comment.