Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set default format of structured dataset to empty #1159

Merged
merged 5 commits into from
Dec 9, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions .github/workflows/pythonbuild.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ on:
pull_request:

env:
FLYTE_SDK_LOGGING_LEVEL: 10 # debug
FLYTE_SDK_LOGGING_LEVEL: 10 # debug

jobs:
build:
Expand Down Expand Up @@ -49,11 +49,11 @@ jobs:
pip freeze
- name: Test with coverage
run: |
pytest tests/flytekit/unit -m "not sandbox_test" --cov=./ --cov-report=xml
make unit_test_codecov
- name: Codecov
uses: codecov/[email protected]
with:
fail_ci_if_error: true # optional (default = false)
fail_ci_if_error: false

build-plugins:
runs-on: ubuntu-latest
Expand All @@ -68,6 +68,7 @@ jobs:
- flytekit-aws-sagemaker
- flytekit-bigquery
- flytekit-data-fsspec
- flytekit-dbt
- flytekit-deck-standard
- flytekit-dolt
- flytekit-greatexpectations
Expand All @@ -87,6 +88,7 @@ jobs:
- flytekit-snowflake
- flytekit-spark
- flytekit-sqlalchemy
- flytekit-vaex
- flytekit-whylogs
exclude:
# flytekit-modin depends on ray which does not have a 3.10 wheel yet.
Expand All @@ -109,8 +111,6 @@ jobs:
# Issue tracked: https://github.com/whylabs/whylogs/issues/697
- python-version: 3.10
plugin-names: "flytekit-whylogs"


steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
Expand All @@ -136,7 +136,6 @@ jobs:
run: |
cd plugins/${{ matrix.plugin-names }}
coverage run -m pytest tests

lint:
runs-on: ubuntu-latest
steps:
Expand Down Expand Up @@ -178,4 +177,6 @@ jobs:
python -m pip install --upgrade pip==21.2.4 setuptools wheel
pip install -r doc-requirements.txt
- name: Build the documentation
run: make -C docs html
run: |
# TODO: Remove after buf migration is done and packages updated
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python make -C docs html
72 changes: 51 additions & 21 deletions .github/workflows/pythonpublish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,31 +65,61 @@ jobs:
- uses: actions/checkout@v2
with:
fetch-depth: "0"
- name: Build & Push Flytekit Python${{ matrix.python-version }} Docker Image to Github Registry
uses: whoan/docker-build-with-cache-action@v5
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v1
- name: Login to GitHub Container Registry
if: ${{ github.event_name == 'release' }}
uses: docker/login-action@v1
with:
# https://docs.github.com/en/packages/learn-github-packages/publishing-a-package
registry: ghcr.io
username: "${{ secrets.FLYTE_BOT_USERNAME }}"
password: "${{ secrets.FLYTE_BOT_PAT }}"
image_name: ${{ github.repository_owner }}/flytekit
image_tag: py${{ matrix.python-version }}-latest,py${{ matrix.python-version }}-${{ github.sha }},py${{ matrix.python-version }}-${{ needs.deploy.outputs.version }}
push_git_tag: true
push_image_and_stages: true
registry: ghcr.io
build_extra_args: "--compress=true --build-arg=VERSION=${{ needs.deploy.outputs.version }} --build-arg=DOCKER_IMAGE=ghcr.io/flyteorg/flytekit:py${{ matrix.python-version }}-${{ needs.deploy.outputs.version }}"
- name: Prepare Flytekit Image Names
id: flytekit-names
uses: docker/metadata-action@v3
with:
images: |
ghcr.io/${{ github.repository_owner }}/flytekit
tags: |
py${{ matrix.python-version }}-latest
py${{ matrix.python-version }}-${{ github.sha }}
py${{ matrix.python-version }}-${{ needs.deploy.outputs.version }}
- name: Build & Push Flytekit Python${{ matrix.python-version }} Docker Image to Github Registry
uses: docker/build-push-action@v2
with:
context: .
dockerfile: Dockerfile.py${{ matrix.python-version }}
platforms: linux/arm64, linux/amd64
push: ${{ github.event_name == 'release' }}
tags: ${{ steps.flytekit-names.outputs.tags }}
build-args: |
VERSION=${{ needs.deploy.outputs.version }}
DOCKER_IMAGE=ghcr.io/${{ github.repository_owner }}/flytekit:py${{ matrix.python-version }}-${{ needs.deploy.outputs.version }}
PYTHON_VERSION=${{ matrix.python-version }}
file: Dockerfile
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Prepare SQLAlchemy Image Names
id: sqlalchemy-names
uses: docker/metadata-action@v3
with:
images: |
ghcr.io/${{ github.repository_owner }}/flytekit
tags: |
py${{ matrix.python-version }}-sqlalchemy-latest
py${{ matrix.python-version }}-sqlalchemy-${{ github.sha }}
py${{ matrix.python-version }}-sqlalchemy-${{ needs.deploy.outputs.version }}
- name: Push SQLAlchemy Image to GitHub Registry
uses: whoan/docker-build-with-cache-action@v5
uses: docker/build-push-action@v2
with:
# https://docs.github.com/en/packages/learn-github-packages/publishing-a-package
username: "${{ secrets.FLYTE_BOT_USERNAME }}"
password: "${{ secrets.FLYTE_BOT_PAT }}"
image_name: ${{ github.repository_owner }}/flytekit
image_tag: py${{ matrix.python-version }}-sqlalchemy-latest,py${{ matrix.python-version }}-sqlalchemy-${{ github.sha }},py${{ matrix.python-version }}-sqlalchemy-${{ needs.deploy.outputs.version }}
push_git_tag: true
push_image_and_stages: true
registry: ghcr.io
build_extra_args: "--compress=true --build-arg=VERSION=${{ needs.deploy.outputs.version }}"
context: "./plugins/flytekit-sqlalchemy/"
dockerfile: Dockerfile.py${{ matrix.python-version }}
platforms: linux/arm64, linux/amd64
push: ${{ github.event_name == 'release' }}
tags: ${{ steps.sqlalchemy-names.outputs.tags }}
build-args: |
VERSION=${{ needs.deploy.outputs.version }}
file: ./plugins/flytekit-sqlalchemy/Dockerfile.py${{ matrix.python-version }}
cache-from: type=gha
cache-to: type=gha,mode=max
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,4 @@ docs/source/plugins/generated/
htmlcov
*.ipynb
*dat
source/_tags/
23 changes: 23 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
ARG PYTHON_VERSION
FROM python:${PYTHON_VERSION}-slim-buster

MAINTAINER Flyte Team <[email protected]>
LABEL org.opencontainers.image.source https://github.com/flyteorg/flytekit

WORKDIR /root
ENV PYTHONPATH /root

ARG VERSION
ARG DOCKER_IMAGE

RUN apt-get update && apt-get install build-essential -y

# Pod tasks should be exposed in the default image
RUN pip install -U flytekit==$VERSION \
flytekitplugins-pod==$VERSION \
flytekitplugins-deck-standard==$VERSION \
flytekitplugins-data-fsspec[aws]==$VERSION \
flytekitplugins-data-fsspec[gcp]==$VERSION \
scikit-learn

ENV FLYTE_INTERNAL_IMAGE "$DOCKER_IMAGE"
18 changes: 0 additions & 18 deletions Dockerfile.py3.10

This file was deleted.

18 changes: 0 additions & 18 deletions Dockerfile.py3.7

This file was deleted.

18 changes: 0 additions & 18 deletions Dockerfile.py3.8

This file was deleted.

18 changes: 0 additions & 18 deletions Dockerfile.py3.9

This file was deleted.

2 changes: 1 addition & 1 deletion MANIFEST.in
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
# include folders
recursive-include flytekit *
recursive-include flytekit_scripts *
recursive-include plugins *

# include specific files
include README.md
Expand All @@ -25,6 +24,7 @@ recursive-exclude tests *
recursive-exclude docs *
recursive-exclude boilerplate *
recursive-exclude .github *
recursive-exclude plugins *

# exclude dist folder:
# - contains the generated *.tar.gz and .whl files.
Expand Down
11 changes: 10 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,18 @@ spellcheck: ## Runs a spellchecker over all code and documentation
.PHONY: test
test: lint unit_test

.PHONY: unit_test_codecov
unit_test_codecov:
# Ensure coverage file
rm coverage.xml || true
$(MAKE) CODECOV_OPTS="--cov=./ --cov-report=xml --cov-append" unit_test

.PHONY: unit_test
unit_test:
pytest -m "not sandbox_test" tests/flytekit/unit
# Skip tensorflow tests and run them with the necessary env var set so that a working (albeit slower)
# library is used to serialize/deserialize protobufs is used.
pytest -m "not sandbox_test" tests/flytekit/unit/ --ignore=tests/flytekit/unit/extras/tensorflow ${CODECOV_OPTS} && \
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python pytest tests/flytekit/unit/extras/tensorflow ${CODECOV_OPTS}

requirements-spark2.txt: export CUSTOM_COMPILE_COMMAND := make requirements-spark2.txt
requirements-spark2.txt: requirements-spark2.in install-piptools
Expand Down
6 changes: 5 additions & 1 deletion dev-requirements.in
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,8 @@ codespell
google-cloud-bigquery
google-cloud-bigquery-storage
IPython
torch
tensorflow==2.8.1
# Newer versions of torch bring in nvidia dependencies that are not present in windows, so
# we put this constraint while we do not have per-environment requirements files
torch<=1.12.1
scikit-learn
Loading