chore: create separate action to setup python and rust in the cicd pipeline

.github/actions/setup-env/action.yml

@@ -0,0 +1,34 @@
+name: "Setup Python and Rust Environment"
+description: "Set up Python, virtual environment, and Rust toolchain"
+
+inputs:
+
+  python-version:
+    description: "The Python version to set up"
+    required: true
+    default: "3.10"
+
+  rust-toolchain:
+    description: "The Rust toolchain to set up"
+    required: true
+    default: "stable"
+
+runs:
+  using: "composite"
+
+  steps:
+
+    - name: Set up Python ${{ inputs.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - name: Install Rust toolchain
+      uses: actions-rs/toolchain@v1
+      with:
+        profile: default
+        toolchain: ${{ inputs.rust-toolchain }}
+        override: true
+        components: rustfmt, clippy
+
+    - uses: Swatinem/rust-cache@v2

.github/workflows/docs.yml

@@ -47,25 +47,13 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-          components: rustfmt, clippy
-
-      - uses: Swatinem/rust-cache@v2
-
-      - name: Set up Python
-        uses: actions/setup-python@v3
-        with:
-          python-version: '3.10'
+      - name: Setup Environment
+        uses: ./.github/actions/setup-env
 
       - name: Build and install deltalake
         run: |
           cd python
-          pip install virtualenv
-          virtualenv venv
+          python -m venv venv
           source venv/bin/activate
           make ${{ env.BUILD_ARGS }}
 

.github/workflows/python_build.yml

@@ -15,23 +15,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Setup Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: "3.10"
+
+      - name: Setup Environment
+        uses: ./.github/actions/setup-env
 
       - name: Check Python
         run: |
+          python -m venv venv
+          source venv/bin/activate
           pip install ruff==0.5.2 mypy==1.10.1 types-dataclasses typing-extensions
           make check-python
 
-      - name: Install minimal stable with clippy and rustfmt
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: default
-          toolchain: stable
-          override: true
-
       - name: Check Rust
         run: make check-rust
 
@@ -45,24 +39,14 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Setup Python
-        uses: actions/setup-python@v2
+      - name: Setup Environment
+        uses: ./.github/actions/setup-env
         with:
           python-version: 3.8
 
-      - name: Install latest nightly
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-          components: rustfmt, clippy
-
-      - uses: Swatinem/rust-cache@v2
-
       - name: Build and install deltalake
         run: |
-          pip install virtualenv
-          virtualenv venv
+          python -m venv venv
           source venv/bin/activate
           make setup
           # Install minimum PyArrow version
@@ -89,26 +73,15 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Install latest nightly
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-          components: rustfmt, clippy
-
-      - uses: Swatinem/rust-cache@v2
-
-      - uses: actions/setup-python@v3
-        with:
-          python-version: "3.10"
+      - name: Setup Environment
+        uses: ./.github/actions/setup-env
 
       - name: Start emulated services
         run: docker-compose up -d
 
       - name: Build and install deltalake
         run: |
-          pip install virtualenv
-          virtualenv venv
+          python -m venv venv
           source venv/bin/activate
           make develop
 
@@ -137,23 +110,12 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Install latest nightly
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-          components: rustfmt, clippy
-
-      - uses: Swatinem/rust-cache@v2
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
+      - name: Setup Environment
+        uses: ./.github/actions/setup-env
 
       - name: Build deltalake in release mode
         run: |
-          pip install virtualenv
-          virtualenv venv
+          python -m venv venv
           source venv/bin/activate
           MATURIN_EXTRA_ARGS=--release make develop
 
@@ -187,18 +149,8 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Install latest nightly
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-          components: rustfmt, clippy
-
-      - uses: Swatinem/rust-cache@v2
-
-      - uses: actions/setup-python@v3
-        with:
-          python-version: "3.10"
+      - name: Setup Environment
+        uses: ./.github/actions/setup-env
 
       - uses: actions/setup-java@v2
         with:
@@ -207,8 +159,7 @@ jobs:
 
       - name: Build and install deltalake
         run: |
-          pip install virtualenv
-          virtualenv venv
+          python -m venv venv
           source venv/bin/activate
           make develop-pyspark
 
@@ -231,15 +182,14 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+      - name: Setup Environment
+        uses: ./.github/actions/setup-env
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Build and install deltalake
         run: |
-          pip install virtualenv
-          virtualenv venv
+          python -m venv venv
           source venv/bin/activate
           make setup
           maturin develop

.gitignore

@@ -21,6 +21,7 @@ __blobstorage__
 .githubchangeloggenerator.cache.log
 .githubchangeloggenerator.cache/
 .githubchangeloggenerator*
+data
 
 # Add all Cargo.lock files except for those in binary crates
 Cargo.lock
@@ -30,4 +31,4 @@ Cargo.lock
 
 justfile
 site
-__pycache__
+__pycache__