From e5204af9c5432cadc3a5f5d3e40ac2b6a7c5937e Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Mon, 14 Aug 2023 13:46:17 -0700 Subject: [PATCH 1/6] add aarch64 workflow --- .../workflows/build_wheels_aarch64_linux.yml | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .github/workflows/build_wheels_aarch64_linux.yml diff --git a/.github/workflows/build_wheels_aarch64_linux.yml b/.github/workflows/build_wheels_aarch64_linux.yml new file mode 100644 index 000000000..613b31e6e --- /dev/null +++ b/.github/workflows/build_wheels_aarch64_linux.yml @@ -0,0 +1,48 @@ +name: Build Linux Wheels + +on: + pull_request: + push: + branches: + - nightly + workflow_dispatch: + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux-aarch64 + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-cuda: disable + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/data + pre-script: packaging/pre_build_script_linux.sh + post-script: packaging/post_build_script_linux.sh + smoke-test-script: test/smoke_test/smoke_test.py + package-name: torchdata + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} + smoke-test-script: ${{ matrix.smoke-test-script }} + package-name: ${{ matrix.package-name }} + env-var-script: packaging/env-var-script.txt + architecture: aarch64 + # Using "development" as trigger event so these binaries are not uploaded to official channels yet + trigger-event: development + secrets: + AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.PYTORCH_BINARY_AWS_ACCESS_KEY_ID }} + AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY }} From 04d3d5bb89e46c46f7819e2a849f76db9405c425 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Mon, 14 Aug 2023 13:48:37 -0700 Subject: [PATCH 2/6] add correct name to workflow --- .github/workflows/build_wheels_aarch64_linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels_aarch64_linux.yml b/.github/workflows/build_wheels_aarch64_linux.yml index 613b31e6e..0e9e38ddc 100644 --- a/.github/workflows/build_wheels_aarch64_linux.yml +++ b/.github/workflows/build_wheels_aarch64_linux.yml @@ -1,4 +1,4 @@ -name: Build Linux Wheels +name: Build AARCH64 Linux Wheels on: pull_request: From 83ee1627a279d3e0e97f46a894b508d9e788150e Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Tue, 15 Aug 2023 10:47:16 -0700 Subject: [PATCH 3/6] set setup-miniconda to false. --- .github/workflows/build_wheels_aarch64_linux.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_wheels_aarch64_linux.yml b/.github/workflows/build_wheels_aarch64_linux.yml index 0e9e38ddc..7b1ffac58 100644 --- a/.github/workflows/build_wheels_aarch64_linux.yml +++ b/.github/workflows/build_wheels_aarch64_linux.yml @@ -41,6 +41,7 @@ jobs: package-name: ${{ matrix.package-name }} env-var-script: packaging/env-var-script.txt architecture: aarch64 + setup-miniconda: false # Using "development" as trigger event so these binaries are not uploaded to official channels yet trigger-event: development secrets: From fed3ae2129441b32e484e05b7aa6e4e79545902a Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Tue, 15 Aug 2023 13:44:25 -0700 Subject: [PATCH 4/6] update post_build_script_linux for mulitple arches --- packaging/post_build_script_linux.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packaging/post_build_script_linux.sh b/packaging/post_build_script_linux.sh index bf7ef1196..2738f638b 100644 --- a/packaging/post_build_script_linux.sh +++ b/packaging/post_build_script_linux.sh @@ -1,14 +1,16 @@ #!/bin/bash set -ex +cpu_arch=`uname -m` + pip3 install auditwheel pkginfo for pkg in dist/torchdata*.whl; do echo "PkgInfo of $pkg:" pkginfo $pkg - auditwheel repair $pkg --plat manylinux2014_x86_64 -w wheelhouse + auditwheel repair $pkg --plat manylinux2014_${cpu_arch} -w wheelhouse - pkg_name=`basename ${pkg%-linux_x86_64.whl}` - auditwheel show wheelhouse/${pkg_name}-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + pkg_name=`basename ${pkg%-linux_${cpu_arch}.whl}` + auditwheel show wheelhouse/${pkg_name}-manylinux_2_17_${cpu_arch}.manylinux2014_${cpu_arch}.whl done From 5d745da8e20e2634f35a9d8d38f03b2b14d1a453 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Tue, 15 Aug 2023 13:55:47 -0700 Subject: [PATCH 5/6] remove torchvision form domain_ci --- .github/workflows/domain_ci.yml | 49 --------------------------------- 1 file changed, 49 deletions(-) diff --git a/.github/workflows/domain_ci.yml b/.github/workflows/domain_ci.yml index 079138389..201573aa7 100644 --- a/.github/workflows/domain_ci.yml +++ b/.github/workflows/domain_ci.yml @@ -10,55 +10,6 @@ on: - gh/*/*/base jobs: - torchvision: - if: ${{ github.repository_owner == 'pytorch' }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: - - macos-latest - - ubuntu-latest - - windows-latest - python-version: - - 3.8 - - 3.9 - steps: - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install torch and torchvision from nightlies - run: | - pip install numpy networkx - pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu - - - name: Check out torchdata repository - uses: actions/checkout@v3 - - - name: Install torchdata - run: | - pip install -r requirements.txt - pip install . - - - name: Install test requirements - run: pip install pytest pytest-mock scipy iopath pycocotools h5py - - - name: Extract torchvision ref - id: torchvision - run: echo "ref=$(python -c 'import torchvision; print(torchvision.version.git_version)')" >> $GITHUB_OUTPUT - - - name: Check out torchvision repository - uses: actions/checkout@v3 - with: - repository: pytorch/vision - ref: ${{ steps.torchvision.outputs.ref }} - path: vision - - - name: Run torchvision builtin datasets tests - run: pytest --no-header -v vision/test/test_prototype_datasets_builtin.py - torchtext: if: ${{ github.repository_owner == 'pytorch' }} runs-on: ${{ matrix.os }} From 29ae9abb26cfaf2b471c316e2a862d93b94db3e8 Mon Sep 17 00:00:00 2001 From: Mike Schneider Date: Tue, 15 Aug 2023 21:51:43 +0000 Subject: [PATCH 6/6] Prettier lint update --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 629146f91..5679e2970 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,11 @@ [**What are DataPipes?**](#what-are-datapipes) | [**Beta Usage and Feedback**](#beta-usage-and-feedback) | [**Contributing**](#contributing) | [**Future Plans**](#future-plans) -**:warning: As of July 2023, we have paused active development on TorchData and have paused new releases. We have learnt a lot from building it and hearing from users, but also believe we need to re-evaluate the technical design and approach given how much the industry has changed since we began the project. During the rest of 2023 we will be re-evaluating our plans in this space. Please reach out if you suggestions or comments (please use [#1196](https://github.com/pytorch/data/issues/1196) for feedback).** +**:warning: As of July 2023, we have paused active development on TorchData and have paused new releases. We have learnt +a lot from building it and hearing from users, but also believe we need to re-evaluate the technical design and approach +given how much the industry has changed since we began the project. During the rest of 2023 we will be re-evaluating our +plans in this space. Please reach out if you suggestions or comments (please use +[#1196](https://github.com/pytorch/data/issues/1196) for feedback).** `torchdata` is a library of common modular data loading primitives for easily constructing flexible and performant data pipelines.