From 3e55e7f6737e5c81d55722e03249aa5834868206 Mon Sep 17 00:00:00 2001
From: Maxime Beauchemin <maximebeauchemin@gmail.com>
Date: Mon, 18 Mar 2024 18:39:41 -0700
Subject: [PATCH] chore: 2nd try - simplify python dependencies (#27505)

---
 .github/actions/setup-backend/action.yml      |  33 ++
 .github/workflows/pre-commit.yml              |  19 +-
 .../workflows/superset-applitool-cypress.yml  |  23 +-
 .github/workflows/superset-cli.yml            |  15 +-
 .github/workflows/superset-e2e.yml            |  12 +-
 .../superset-python-integrationtest.yml       |  30 +-
 .github/workflows/superset-python-misc.yml    |  25 +-
 .../workflows/superset-python-presto-hive.yml |  30 +-
 .../workflows/superset-python-unittest.yml    |  16 +-
 .github/workflows/superset-translations.yml   |   2 +-
 CONTRIBUTING.md                               |   6 +-
 Dockerfile                                    |  11 +-
 Makefile                                      |   6 +-
 UPDATING.md                                   |   5 +-
 docs/docs/contributing/hooks-and-linting.mdx  |   2 +-
 docs/docs/contributing/local-backend.mdx      |   4 +-
 requirements/base.txt                         | 423 ++++++++++++++----
 requirements/development.in                   |  18 +-
 requirements/development.txt                  | 198 +++++++-
 requirements/docker.in                        |  19 -
 requirements/docker.txt                       |  23 -
 requirements/integration.in                   |  19 -
 requirements/integration.txt                  |  74 ---
 requirements/local.in                         |  17 -
 requirements/local.txt                        |  15 -
 requirements/testing.in                       |  31 --
 requirements/testing.txt                      | 147 ------
 tox.ini                                       |   8 +-
 28 files changed, 621 insertions(+), 610 deletions(-)
 create mode 100644 .github/actions/setup-backend/action.yml
 delete mode 100644 requirements/docker.in
 delete mode 100644 requirements/docker.txt
 delete mode 100644 requirements/integration.in
 delete mode 100644 requirements/integration.txt
 delete mode 100644 requirements/local.in
 delete mode 100644 requirements/local.txt
 delete mode 100644 requirements/testing.in
 delete mode 100644 requirements/testing.txt

diff --git a/.github/actions/setup-backend/action.yml b/.github/actions/setup-backend/action.yml
new file mode 100644
index 0000000000000..d3624bf531e89
--- /dev/null
+++ b/.github/actions/setup-backend/action.yml
@@ -0,0 +1,33 @@
+name: 'Setup Python Environment'
+description: 'Set up Python and install dependencies with optional configurations.'
+inputs:
+  python-version:
+    description: 'Python version to set up.'
+    required: true
+    default: '3.9'
+  cache:
+    description: 'Cache dependencies. Options: pip'
+    required: false
+    default: 'pip'
+  requirements-type:
+    description: 'Type of requirements to install. Options: base, development, default'
+    required: false
+    default: 'dev'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Set up Python ${{ inputs.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+        cache: ${{ inputs.cache }}
+    - name: Install dependencies
+      run: |
+        pip install --upgrade pip setuptools wheel
+        if [ "${{ inputs.requirements-type }}" = "dev" ]; then
+          pip install -r requirements/development.txt
+        elif [ "${{ inputs.requirements-type }}" = "base" ]; then
+          pip install -r requirements/base.txt
+        fi
+      shell: bash
diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
index f9a3681ce0c12..2d377accdf0fc 100644
--- a/.github/workflows/pre-commit.yml
+++ b/.github/workflows/pre-commit.yml
@@ -26,24 +26,9 @@ jobs:
           persist-credentials: false
           submodules: recursive
       - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: |
-            requirements/base.txt
-            requirements/integration.txt
-      - name: Install dependencies
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/base.txt
-            pip install -r requirements/integration.txt
-      # Add brew to the path - see https://github.com/actions/runner-images/issues/6283
+        uses: ./.github/actions/setup-backend/
       - name: Enable brew and helm-docs
+        # Add brew to the path - see https://github.com/actions/runner-images/issues/6283
         run: |
           echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH
           eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
diff --git a/.github/workflows/superset-applitool-cypress.yml b/.github/workflows/superset-applitool-cypress.yml
index 7341372cadff8..fa5da3ee9cec8 100644
--- a/.github/workflows/superset-applitool-cypress.yml
+++ b/.github/workflows/superset-applitool-cypress.yml
@@ -58,23 +58,7 @@ jobs:
           submodules: recursive
           ref: master
       - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.9"
-      - name: OS dependencies
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: apt-get-install
-      - name: Install python dependencies
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            pip-upgrade
-            pip install -r requirements/testing.txt
-      - name: Setup postgres
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: setup-postgres
+        uses: ./.github/actions/setup-backend/
       - name: Import test data
         uses: ./.github/actions/cached-dependencies
         with:
@@ -91,6 +75,11 @@ jobs:
         uses: ./.github/actions/cached-dependencies
         with:
           run: build-instrumented-assets
+      - name: Setup Postgres
+        if: steps.check.outcome == 'failure'
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: setup-postgres
       - name: Install cypress
         uses: ./.github/actions/cached-dependencies
         with:
diff --git a/.github/workflows/superset-cli.yml b/.github/workflows/superset-cli.yml
index 9c52f3649769c..07aa01ebfa802 100644
--- a/.github/workflows/superset-cli.yml
+++ b/.github/workflows/superset-cli.yml
@@ -52,22 +52,13 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
+      - name: Setup Postgres
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
-            setup-postgres
+          run: setup-postgres
       - name: superset init
         if: steps.check.outcome == 'failure'
         run: |
diff --git a/.github/workflows/superset-e2e.yml b/.github/workflows/superset-e2e.yml
index 71b415ee6e2c0..4c2423238bc38 100644
--- a/.github/workflows/superset-e2e.yml
+++ b/.github/workflows/superset-e2e.yml
@@ -69,21 +69,15 @@ jobs:
         run: ./scripts/ci_check_no_file_changes.sh python frontend
       - name: Setup Python
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.9"
+        uses: ./.github/actions/setup-backend/
       - name: OS dependencies
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: apt-get-install
-      - name: Install python dependencies
+      - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            pip-upgrade
-            pip install -r requirements/testing.txt
       - name: Setup postgres
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
diff --git a/.github/workflows/superset-python-integrationtest.yml b/.github/workflows/superset-python-integrationtest.yml
index 6bf314d961ffb..28df703a92e9b 100644
--- a/.github/workflows/superset-python-integrationtest.yml
+++ b/.github/workflows/superset-python-integrationtest.yml
@@ -51,21 +51,15 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
+      - name: Setup MySQL
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
             setup-mysql
       - name: Run celery
         if: steps.check.outcome == 'failure'
@@ -78,7 +72,6 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: |
           bash .github/workflows/codecov.sh -c -F python -F mysql
-
   test-postgres:
     runs-on: ubuntu-20.04
     strategy:
@@ -117,21 +110,15 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
+      - name: Setup Postgres
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
             setup-postgres
       - name: Run celery
         if: steps.check.outcome == 'failure'
@@ -177,21 +164,16 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
       - name: Install dependencies
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
+            # sqlite needs this working directory
             mkdir ${{ github.workspace }}/.temp
       - name: Run celery
         if: steps.check.outcome == 'failure'
diff --git a/.github/workflows/superset-python-misc.yml b/.github/workflows/superset-python-misc.yml
index e05e05f48bc46..1c7b6b6534941 100644
--- a/.github/workflows/superset-python-misc.yml
+++ b/.github/workflows/superset-python-misc.yml
@@ -38,21 +38,10 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
-        if: steps.check.outcome == 'failure'
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
       - name: pylint
         if: steps.check.outcome == 'failure'
         # `-j 0` run Pylint in parallel
@@ -70,18 +59,8 @@ jobs:
           persist-credentials: false
           submodules: recursive
       - name: Setup Python
-        uses: actions/setup-python@v5
+        uses: ./.github/actions/setup-backend/
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/base.txt"
-      - name: Install dependencies
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/base.txt
       - name: Test babel extraction
         run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct
diff --git a/.github/workflows/superset-python-presto-hive.yml b/.github/workflows/superset-python-presto-hive.yml
index af2c844057dd9..dd8a839d62647 100644
--- a/.github/workflows/superset-python-presto-hive.yml
+++ b/.github/workflows/superset-python-presto-hive.yml
@@ -67,22 +67,13 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
+      - name: Setup Postgres
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
-            setup-postgres
+          run: setup-postgres
       - name: Run celery
         if: steps.check.outcome == 'failure'
         run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
@@ -144,22 +135,13 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: docker compose -f scripts/databases/hive/docker-compose.yml up -d
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
+      - name: Setup Postgres
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
-            setup-postgres
+          run: setup-postgres
       - name: Run celery
         if: steps.check.outcome == 'failure'
         run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
diff --git a/.github/workflows/superset-python-unittest.yml b/.github/workflows/superset-python-unittest.yml
index 2bdb8cef3e5ed..535cf0335f291 100644
--- a/.github/workflows/superset-python-unittest.yml
+++ b/.github/workflows/superset-python-unittest.yml
@@ -46,27 +46,15 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      # TODO: separated requirements.txt file just for unit tests
-      - name: Install dependencies
-        if: steps.check.outcome == 'failure'
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
-            mkdir ${{ github.workspace }}/.temp
       - name: Python unit tests
         if: steps.check.outcome == 'failure'
         env:
           SUPERSET_TESTENV: true
+          SUPERSET_SECRET_KEY: not-a-secret
         run: |
           pytest --durations-min=0.5 --cov-report= --cov=superset ./tests/common ./tests/unit_tests --cache-clear
       - name: Upload code coverage
diff --git a/.github/workflows/superset-translations.yml b/.github/workflows/superset-translations.yml
index 4a3d9cc21a106..fb00f9f2eabb6 100644
--- a/.github/workflows/superset-translations.yml
+++ b/.github/workflows/superset-translations.yml
@@ -47,7 +47,7 @@ jobs:
           persist-credentials: false
           submodules: recursive
       - name: Setup Python
-        uses: actions/setup-python@v5
+        uses: ./.github/actions/setup-backend/
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 4afa69eb2e251..50c0663972a05 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -477,7 +477,7 @@ python3 -m venv venv # setup a python3 virtualenv
 source venv/bin/activate
 
 # Install external dependencies
-pip install -r requirements/testing.txt
+pip install -r requirements/development.txt
 
 # Install Superset in editable (development) mode
 pip install -e .
@@ -531,7 +531,7 @@ If you add a new requirement or update an existing requirement (per the `install
 ```bash
 $ python3 -m venv venv
 $ source venv/bin/activate
-$ python3 -m pip install -r requirements/integration.txt
+$ python3 -m pip install -r requirements/development.txt
 $ pip-compile-multi --no-upgrade
 ```
 
@@ -749,7 +749,7 @@ The current status of the usability of each flag (stable vs testing, etc) can be
 Superset uses Git pre-commit hooks courtesy of [pre-commit](https://pre-commit.com/). To install run the following:
 
 ```bash
-pip3 install -r requirements/integration.txt
+pip3 install -r requirements/development.txt
 pre-commit install
 ```
 
diff --git a/Dockerfile b/Dockerfile
index 515e823e7297a..e5387d677944a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -79,12 +79,10 @@ RUN mkdir -p ${PYTHONPATH} superset/static superset-frontend apache_superset.egg
 COPY --chown=superset:superset setup.py MANIFEST.in README.md ./
 # setup.py uses the version information in package.json
 COPY --chown=superset:superset superset-frontend/package.json superset-frontend/
-RUN --mount=type=bind,target=./requirements/local.txt,src=./requirements/local.txt \
-    --mount=type=bind,target=./requirements/development.txt,src=./requirements/development.txt \
-    --mount=type=bind,target=./requirements/base.txt,src=./requirements/base.txt \
+RUN --mount=type=bind,target=./requirements/base.txt,src=./requirements/base.txt \
     --mount=type=cache,target=/root/.cache/pip \
     pip install --upgrade setuptools pip && \
-    pip install -r requirements/local.txt
+    pip install -r requirements/base.txt
 
 COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets
 ## Lastly, let's install superset itself
@@ -128,10 +126,9 @@ RUN apt-get update -qq \
     && ln -s /opt/firefox/firefox /usr/local/bin/firefox \
     && apt-get autoremove -yqq --purge wget && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/*
 # Cache everything for dev purposes...
-RUN --mount=type=bind,target=./requirements/base.txt,src=./requirements/base.txt \
-    --mount=type=bind,target=./requirements/docker.txt,src=./requirements/docker.txt \
+RUN --mount=type=bind,target=./requirements/development.txt,src=./requirements/development.txt \
     --mount=type=cache,target=/root/.cache/pip \
-    pip install -r requirements/docker.txt
+    pip install -r requirements/development.txt
 
 USER superset
 ######################################################################
diff --git a/Makefile b/Makefile
index 14e7bb17efdac..42ec9f5aa4a10 100644
--- a/Makefile
+++ b/Makefile
@@ -24,7 +24,7 @@ install: superset pre-commit
 
 superset:
 	# Install external dependencies
-	pip install -r requirements/local.txt
+	pip install -r requirements/development.txt
 
 	# Install Superset in editable (development) mode
 	pip install -e .
@@ -53,7 +53,7 @@ update: update-py update-js
 
 update-py:
 	# Install external dependencies
-	pip install -r requirements/local.txt
+	pip install -r requirements/development.txt
 
 	# Install Superset in editable (development) mode
 	pip install -e .
@@ -79,7 +79,7 @@ activate:
 
 pre-commit:
 	# setup pre commit dependencies
-	pip3 install -r requirements/integration.txt
+	pip3 install -r requirements/development.txt
 	pre-commit install
 
 format: py-format js-format
diff --git a/UPDATING.md b/UPDATING.md
index c91de826bda15..b035b963f2d65 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -23,7 +23,10 @@ This file documents any backwards-incompatible changes in Superset and
 assists people when migrating to a new version.
 
 ## Next
-
+- [27505](https://github.com/apache/superset/pull/27505): We simplified the files under
+  `requirements/` folder. If you use these files for your builds you may want to double
+  check that your builds are not affected. `base.txt` should be the same as before, though
+  `development.txt` becomes a bigger set, incorporating the now defunct local,testing,integration, and docker
 - [27119](https://github.com/apache/superset/pull/27119): Updates various database columns to use the `MediumText` type, potentially requiring a table lock on MySQL dbs or taking some time to complete on large deployments.
 
 - [26450](https://github.com/apache/superset/pull/26450): Deprecates the `KV_STORE` feature flag and its related assets such as the API endpoint and `keyvalue` table. The main dependency of this feature is the `SHARE_QUERIES_VIA_KV_STORE` feature flag which allows sharing SQL Lab queries without the necessity of saving the query. Our intention is to use the permalink feature to implement this use case before 5.0 and that's why we are deprecating the feature flag now.
diff --git a/docs/docs/contributing/hooks-and-linting.mdx b/docs/docs/contributing/hooks-and-linting.mdx
index 68c9d28ca00b8..8b4d173c0c3fd 100644
--- a/docs/docs/contributing/hooks-and-linting.mdx
+++ b/docs/docs/contributing/hooks-and-linting.mdx
@@ -10,7 +10,7 @@ version: 1
 Superset uses Git pre-commit hooks courtesy of [pre-commit](https://pre-commit.com/). To install run the following:
 
 ```bash
-pip3 install -r requirements/integration.txt
+pip3 install -r requirements/development.txt
 pre-commit install
 ```
 
diff --git a/docs/docs/contributing/local-backend.mdx b/docs/docs/contributing/local-backend.mdx
index b7d0d8f14af09..a288bd937ca99 100644
--- a/docs/docs/contributing/local-backend.mdx
+++ b/docs/docs/contributing/local-backend.mdx
@@ -20,7 +20,7 @@ python3 -m venv venv # setup a python3 virtualenv
 source venv/bin/activate
 
 # Install external dependencies
-pip install -r requirements/testing.txt
+pip install -r requirements/development.txt
 
 # Install Superset in editable (development) mode
 pip install -e .
@@ -75,7 +75,7 @@ If you add a new requirement or update an existing requirement (per the `install
 ```bash
 $ python3 -m venv venv
 $ source venv/bin/activate
-$ python3 -m pip install -r requirements/integration.txt
+$ python3 -m pip install -r requirements/development.txt
 $ pip-compile-multi --no-upgrade
 ```
 
diff --git a/requirements/base.txt b/requirements/base.txt
index 77521e92303de..3ec5476cb21fe 100644
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -6,54 +6,92 @@
 #    pip-compile-multi
 #
 -e file:.
-    # via -r requirements/base.in
+    # via
+    #   -c requirements/development.txt
+    #   -r requirements/base.in
 alembic==1.6.5
-    # via flask-migrate
+    # via
+    #   -c requirements/development.txt
+    #   flask-migrate
 amqp==5.1.1
-    # via kombu
+    # via
+    #   -c requirements/development.txt
+    #   kombu
 apispec[yaml]==6.3.0
-    # via flask-appbuilder
+    # via
+    #   -c requirements/development.txt
+    #   flask-appbuilder
 apsw==3.42.0.1
-    # via shillelagh
+    # via
+    #   -c requirements/development.txt
+    #   shillelagh
 async-timeout==4.0.2
-    # via redis
+    # via
+    #   -c requirements/development.txt
+    #   redis
 attrs==23.1.0
     # via
+    #   -c requirements/development.txt
     #   cattrs
     #   jsonschema
     #   requests-cache
 babel==2.9.1
-    # via flask-babel
+    # via
+    #   -c requirements/development.txt
+    #   flask-babel
 backoff==1.11.1
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 bcrypt==4.0.1
-    # via paramiko
+    # via
+    #   -c requirements/development.txt
+    #   paramiko
 billiard==4.2.0
-    # via celery
+    # via
+    #   -c requirements/development.txt
+    #   celery
 bottleneck==1.3.7
-    # via pandas
+    # via
+    #   -c requirements/development.txt
+    #   pandas
 brotli==1.0.9
-    # via flask-compress
+    # via
+    #   -c requirements/development.txt
+    #   flask-compress
 cachelib==0.9.0
     # via
+    #   -c requirements/development.txt
     #   flask-caching
     #   flask-session
 cachetools==5.3.2
-    # via google-auth
+    # via
+    #   -c requirements/development.txt
+    #   google-auth
 cattrs==23.2.1
-    # via requests-cache
+    # via
+    #   -c requirements/development.txt
+    #   requests-cache
 celery==5.3.6
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 certifi==2023.7.22
-    # via requests
+    # via
+    #   -c requirements/development.txt
+    #   requests
 cffi==1.15.1
     # via
+    #   -c requirements/development.txt
     #   cryptography
     #   pynacl
 charset-normalizer==3.2.0
-    # via requests
+    # via
+    #   -c requirements/development.txt
+    #   requests
 click==8.1.3
     # via
+    #   -c requirements/development.txt
     #   apache-superset
     #   celery
     #   click-didyoumean
@@ -63,37 +101,62 @@ click==8.1.3
     #   flask
     #   flask-appbuilder
 click-didyoumean==0.3.0
-    # via celery
+    # via
+    #   -c requirements/development.txt
+    #   celery
 click-option-group==0.5.5
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 click-plugins==1.1.1
-    # via celery
+    # via
+    #   -c requirements/development.txt
+    #   celery
 click-repl==0.2.0
-    # via celery
+    # via
+    #   -c requirements/development.txt
+    #   celery
 colorama==0.4.6
     # via
+    #   -c requirements/development.txt
     #   apache-superset
     #   flask-appbuilder
 cron-descriptor==1.2.24
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 croniter==1.0.15
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 cryptography==42.0.4
     # via
+    #   -c requirements/development.txt
     #   apache-superset
     #   paramiko
 deprecated==1.2.13
-    # via limits
+    # via
+    #   -c requirements/development.txt
+    #   limits
 deprecation==2.1.0
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 dnspython==2.1.0
-    # via email-validator
+    # via
+    #   -c requirements/development.txt
+    #   email-validator
 email-validator==1.1.3
-    # via flask-appbuilder
+    # via
+    #   -c requirements/development.txt
+    #   flask-appbuilder
 exceptiongroup==1.2.0
-    # via cattrs
+    # via
+    #   -c requirements/development.txt
+    #   cattrs
 flask==2.2.5
     # via
+    #   -c requirements/development.txt
     #   apache-superset
     #   flask-appbuilder
     #   flask-babel
@@ -107,118 +170,194 @@ flask==2.2.5
     #   flask-sqlalchemy
     #   flask-wtf
 flask-appbuilder==4.4.1
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 flask-babel==1.0.0
-    # via flask-appbuilder
+    # via
+    #   -c requirements/development.txt
+    #   flask-appbuilder
 flask-caching==2.1.0
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 flask-compress==1.13
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 flask-jwt-extended==4.3.1
-    # via flask-appbuilder
+    # via
+    #   -c requirements/development.txt
+    #   flask-appbuilder
 flask-limiter==3.3.1
-    # via flask-appbuilder
+    # via
+    #   -c requirements/development.txt
+    #   flask-appbuilder
 flask-login==0.6.3
     # via
+    #   -c requirements/development.txt
     #   apache-superset
     #   flask-appbuilder
 flask-migrate==3.1.0
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 flask-session==0.5.0
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 flask-sqlalchemy==2.5.1
     # via
+    #   -c requirements/development.txt
     #   flask-appbuilder
     #   flask-migrate
 flask-talisman==1.0.0
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 flask-wtf==1.2.1
     # via
+    #   -c requirements/development.txt
     #   apache-superset
     #   flask-appbuilder
 func-timeout==4.3.5
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 geographiclib==1.52
-    # via geopy
+    # via
+    #   -c requirements/development.txt
+    #   geopy
 geopy==2.2.0
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 google-auth==2.27.0
-    # via shillelagh
+    # via
+    #   -c requirements/development.txt
+    #   shillelagh
 greenlet==3.0.3
-    # via shillelagh
+    # via
+    #   -c requirements/development.txt
+    #   shillelagh
 gunicorn==21.2.0
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 hashids==1.3.1
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 holidays==0.25
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 humanize==3.11.0
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 idna==3.2
     # via
+    #   -c requirements/development.txt
     #   email-validator
     #   requests
 importlib-metadata==6.6.0
     # via
+    #   -c requirements/development.txt
     #   apache-superset
     #   flask
     #   shillelagh
 importlib-resources==5.12.0
-    # via limits
+    # via
+    #   -c requirements/development.txt
+    #   limits
 isodate==0.6.0
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 itsdangerous==2.1.2
     # via
+    #   -c requirements/development.txt
     #   flask
     #   flask-wtf
 jinja2==3.1.3
     # via
+    #   -c requirements/development.txt
     #   flask
     #   flask-babel
 jsonschema==4.17.3
-    # via flask-appbuilder
+    # via
+    #   -c requirements/development.txt
+    #   flask-appbuilder
 kombu==5.3.4
-    # via celery
+    # via
+    #   -c requirements/development.txt
+    #   celery
 korean-lunar-calendar==0.3.1
-    # via holidays
+    # via
+    #   -c requirements/development.txt
+    #   holidays
 limits==3.4.0
-    # via flask-limiter
+    # via
+    #   -c requirements/development.txt
+    #   flask-limiter
 llvmlite==0.40.1
-    # via numba
+    # via
+    #   -c requirements/development.txt
+    #   numba
 mako==1.2.4
     # via
+    #   -c requirements/development.txt
     #   alembic
     #   apache-superset
 markdown==3.3.4
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 markdown-it-py==2.2.0
-    # via rich
+    # via
+    #   -c requirements/development.txt
+    #   rich
 markupsafe==2.1.1
     # via
+    #   -c requirements/development.txt
     #   jinja2
     #   mako
     #   werkzeug
     #   wtforms
 marshmallow==3.19.0
     # via
+    #   -c requirements/development.txt
     #   flask-appbuilder
     #   marshmallow-sqlalchemy
 marshmallow-sqlalchemy==0.23.1
-    # via flask-appbuilder
+    # via
+    #   -c requirements/development.txt
+    #   flask-appbuilder
 mdurl==0.1.2
-    # via markdown-it-py
+    # via
+    #   -c requirements/development.txt
+    #   markdown-it-py
 msgpack==1.0.2
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 nh3==0.2.11
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 numba==0.57.1
-    # via pandas
+    # via
+    #   -c requirements/development.txt
+    #   pandas
 numexpr==2.9.0
     # via
+    #   -c requirements/development.txt
     #   -r requirements/base.in
     #   pandas
 numpy==1.23.5
     # via
+    #   -c requirements/development.txt
     #   apache-superset
     #   bottleneck
     #   numba
@@ -226,9 +365,12 @@ numpy==1.23.5
     #   pandas
     #   pyarrow
 ordered-set==4.1.0
-    # via flask-limiter
+    # via
+    #   -c requirements/development.txt
+    #   flask-limiter
 packaging==23.1
     # via
+    #   -c requirements/development.txt
     #   apache-superset
     #   apispec
     #   deprecation
@@ -237,48 +379,80 @@ packaging==23.1
     #   marshmallow
     #   shillelagh
 pandas[performance]==2.0.3
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 paramiko==3.4.0
     # via
+    #   -c requirements/development.txt
     #   apache-superset
     #   sshtunnel
 parsedatetime==2.6
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 pgsanity==0.2.9
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 platformdirs==3.8.1
-    # via requests-cache
+    # via
+    #   -c requirements/development.txt
+    #   requests-cache
 polyline==2.0.0
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 prison==0.2.1
-    # via flask-appbuilder
+    # via
+    #   -c requirements/development.txt
+    #   flask-appbuilder
 prompt-toolkit==3.0.38
-    # via click-repl
+    # via
+    #   -c requirements/development.txt
+    #   click-repl
 pyarrow==14.0.1
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 pyasn1==0.5.1
     # via
+    #   -c requirements/development.txt
     #   pyasn1-modules
     #   rsa
 pyasn1-modules==0.3.0
-    # via google-auth
+    # via
+    #   -c requirements/development.txt
+    #   google-auth
 pycparser==2.20
-    # via cffi
+    # via
+    #   -c requirements/development.txt
+    #   cffi
 pygments==2.15.0
-    # via rich
+    # via
+    #   -c requirements/development.txt
+    #   rich
 pyjwt==2.4.0
     # via
+    #   -c requirements/development.txt
     #   apache-superset
     #   flask-appbuilder
     #   flask-jwt-extended
 pynacl==1.5.0
-    # via paramiko
+    # via
+    #   -c requirements/development.txt
+    #   paramiko
 pyparsing==3.0.6
-    # via apache-superset
-pyrsistent==0.19.3
-    # via jsonschema
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
+pyrsistent==0.20.0
+    # via
+    #   -c requirements/development.txt
+    #   jsonschema
 python-dateutil==2.8.2
     # via
+    #   -c requirements/development.txt
     #   alembic
     #   apache-superset
     #   celery
@@ -288,42 +462,68 @@ python-dateutil==2.8.2
     #   pandas
     #   shillelagh
 python-dotenv==0.19.0
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 python-editor==1.0.4
-    # via alembic
+    # via
+    #   -c requirements/development.txt
+    #   alembic
 python-geohash==0.8.5
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 pytz==2021.3
     # via
+    #   -c requirements/development.txt
     #   babel
     #   flask-babel
     #   pandas
 pyyaml==6.0.1
     # via
+    #   -c requirements/development.txt
     #   apache-superset
     #   apispec
 redis==4.6.0
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 requests==2.31.0
     # via
+    #   -c requirements/development.txt
     #   requests-cache
     #   shillelagh
 requests-cache==1.1.1
-    # via shillelagh
+    # via
+    #   -c requirements/development.txt
+    #   shillelagh
 rich==13.3.4
-    # via flask-limiter
+    # via
+    #   -c requirements/development.txt
+    #   flask-limiter
 rsa==4.9
-    # via google-auth
+    # via
+    #   -c requirements/development.txt
+    #   google-auth
 selenium==3.141.0
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 shillelagh[gsheetsapi]==1.2.10
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 shortid==0.1.2
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 simplejson==3.17.3
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 six==1.16.0
     # via
+    #   -c requirements/development.txt
     #   click-repl
     #   isodate
     #   prison
@@ -331,9 +531,12 @@ six==1.16.0
     #   url-normalize
     #   wtforms-json
 slack-sdk==3.21.3
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 sqlalchemy==1.4.36
     # via
+    #   -c requirements/development.txt
     #   alembic
     #   apache-superset
     #   flask-appbuilder
@@ -343,18 +546,28 @@ sqlalchemy==1.4.36
     #   sqlalchemy-utils
 sqlalchemy-utils==0.38.3
     # via
+    #   -c requirements/development.txt
     #   apache-superset
     #   flask-appbuilder
 sqlglot==20.8.0
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 sqlparse==0.4.4
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 sshtunnel==0.4.0
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 tabulate==0.8.9
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 typing-extensions==4.4.0
     # via
+    #   -c requirements/development.txt
     #   apache-superset
     #   cattrs
     #   flask-limiter
@@ -363,44 +576,60 @@ typing-extensions==4.4.0
     #   shillelagh
 tzdata==2023.3
     # via
+    #   -c requirements/development.txt
     #   celery
     #   pandas
 url-normalize==1.4.3
-    # via requests-cache
+    # via
+    #   -c requirements/development.txt
+    #   requests-cache
 urllib3==1.26.18
     # via
+    #   -c requirements/development.txt
     #   -r requirements/base.in
     #   requests
     #   requests-cache
     #   selenium
 vine==5.1.0
     # via
+    #   -c requirements/development.txt
     #   amqp
     #   celery
     #   kombu
 wcwidth==0.2.5
-    # via prompt-toolkit
+    # via
+    #   -c requirements/development.txt
+    #   prompt-toolkit
 werkzeug==3.0.1
     # via
+    #   -c requirements/development.txt
     #   -r requirements/base.in
     #   flask
     #   flask-appbuilder
     #   flask-jwt-extended
     #   flask-login
 wrapt==1.15.0
-    # via deprecated
+    # via
+    #   -c requirements/development.txt
+    #   deprecated
 wtforms==2.3.3
     # via
+    #   -c requirements/development.txt
     #   apache-superset
     #   flask-appbuilder
     #   flask-wtf
     #   wtforms-json
 wtforms-json==0.3.5
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 xlsxwriter==3.0.7
-    # via apache-superset
+    # via
+    #   -c requirements/development.txt
+    #   apache-superset
 zipp==3.15.0
     # via
+    #   -c requirements/development.txt
     #   importlib-metadata
     #   importlib-resources
 
diff --git a/requirements/development.in b/requirements/development.in
index 102a13022feda..abd40746767fd 100644
--- a/requirements/development.in
+++ b/requirements/development.in
@@ -17,10 +17,26 @@
 # under the License.
 #
 -r base.in
--e .[cors,druid,hive,mysql,postgres,thumbnails]
+-e .[bigquery,cors,druid,gevent,gsheets,hive,mysql,playwright,postgres,presto,prophet,trino,thumbnails]
+
+docker
+flask-testing
+freezegun
+greenlet>=2.0.2
+grpcio>=1.55.3
 ipython
+openapi-spec-validator
+parameterized
+pip-compile-multi
+pre-commit
 progress>=1.5,<2
+pyfakefs
 pyinstrument>=4.0.2,<5
 pylint
+pytest
+pytest-cov
+pytest-mock
 python-ldap>=3.4.3
 sqloxide
+statsd
+tox
diff --git a/requirements/development.txt b/requirements/development.txt
index c4c6ee9840b32..18da59a79053f 100644
--- a/requirements/development.txt
+++ b/requirements/development.txt
@@ -1,4 +1,4 @@
-# SHA1:5d8f94148b758249eecc1e6a9b2f3ffde15815ad
+# SHA1:3b6a7d105f9d14b449d4232aa368bd6a40d4c7ef
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -24,24 +24,100 @@ botocore==1.29.130
     # via
     #   boto3
     #   s3transfer
+build==0.10.0
+    # via pip-tools
 cached-property==1.5.2
     # via tableschema
+cfgv==3.3.1
+    # via pre-commit
 chardet==5.1.0
-    # via tabulator
+    # via
+    #   tabulator
+    #   tox
+cmdstanpy==1.1.0
+    # via prophet
+contourpy==1.0.7
+    # via matplotlib
+coverage[toml]==7.2.5
+    # via pytest-cov
+cycler==0.11.0
+    # via matplotlib
+db-dtypes==1.1.1
+    # via pandas-gbq
 decorator==5.1.1
     # via ipython
 dill==0.3.6
     # via pylint
+distlib==0.3.6
+    # via virtualenv
+docker==6.1.1
+    # via -r requirements/development.in
 et-xmlfile==1.1.0
     # via openpyxl
 executing==1.2.0
     # via stack-data
+filelock==3.12.2
+    # via
+    #   tox
+    #   virtualenv
 flask-cors==3.0.10
     # via apache-superset
+flask-testing==0.8.1
+    # via -r requirements/development.in
+fonttools==4.43.0
+    # via matplotlib
+freezegun==1.2.2
+    # via -r requirements/development.in
 future==0.18.3
-    # via pyhive
+    # via
+    #   pyhive
+    #   sqlalchemy-bigquery
+gevent==23.9.1
+    # via apache-superset
+google-api-core[grpc]==2.11.0
+    # via
+    #   google-cloud-bigquery
+    #   google-cloud-bigquery-storage
+    #   google-cloud-core
+    #   pandas-gbq
+    #   sqlalchemy-bigquery
+google-auth-oauthlib==1.0.0
+    # via
+    #   pandas-gbq
+    #   pydata-google-auth
+google-cloud-bigquery==3.10.0
+    # via
+    #   apache-superset
+    #   pandas-gbq
+    #   sqlalchemy-bigquery
+google-cloud-bigquery-storage==2.19.1
+    # via
+    #   pandas-gbq
+    #   sqlalchemy-bigquery
+google-cloud-core==2.3.2
+    # via google-cloud-bigquery
+google-crc32c==1.5.0
+    # via google-resumable-media
+google-resumable-media==2.5.0
+    # via google-cloud-bigquery
+googleapis-common-protos==1.59.0
+    # via
+    #   google-api-core
+    #   grpcio-status
+grpcio==1.60.1
+    # via
+    #   -r requirements/development.in
+    #   google-api-core
+    #   google-cloud-bigquery
+    #   grpcio-status
+grpcio-status==1.60.1
+    # via google-api-core
+identify==2.5.24
+    # via pre-commit
 ijson==3.2.0.post0
     # via tabulator
+iniconfig==2.0.0
+    # via pytest
 ipython==8.12.2
     # via -r requirements/development.in
 isort==5.12.0
@@ -54,28 +130,78 @@ jmespath==1.0.1
     #   botocore
 jsonlines==3.1.0
     # via tabulator
+jsonschema-spec==0.1.4
+    # via openapi-spec-validator
+kiwisolver==1.4.4
+    # via matplotlib
 lazy-object-proxy==1.9.0
-    # via astroid
+    # via
+    #   astroid
+    #   openapi-spec-validator
 linear-tsv==1.1.0
     # via tabulator
+matplotlib==3.7.1
+    # via prophet
 matplotlib-inline==0.1.6
     # via ipython
 mccabe==0.7.0
     # via pylint
 mysqlclient==2.1.0
     # via apache-superset
+nodeenv==1.7.0
+    # via pre-commit
+oauthlib==3.2.2
+    # via requests-oauthlib
+openapi-schema-validator==0.4.4
+    # via openapi-spec-validator
+openapi-spec-validator==0.5.6
+    # via -r requirements/development.in
 openpyxl==3.1.2
     # via tabulator
+pandas-gbq==0.19.1
+    # via apache-superset
+parameterized==0.9.0
+    # via -r requirements/development.in
 parso==0.8.3
     # via jedi
+pathable==0.4.3
+    # via jsonschema-spec
 pexpect==4.8.0
     # via ipython
 pickleshare==0.7.5
     # via ipython
 pillow==10.2.0
+    # via
+    #   apache-superset
+    #   matplotlib
+pip-compile-multi==2.6.3
+    # via -r requirements/development.in
+pip-tools==7.3.0
+    # via pip-compile-multi
+playwright==1.41.2
     # via apache-superset
+pluggy==1.2.0
+    # via
+    #   pytest
+    #   tox
+pre-commit==3.3.3
+    # via -r requirements/development.in
 progress==1.6
     # via -r requirements/development.in
+prophet==1.1.5
+    # via apache-superset
+proto-plus==1.22.2
+    # via
+    #   google-cloud-bigquery
+    #   google-cloud-bigquery-storage
+protobuf==4.23.0
+    # via
+    #   google-api-core
+    #   google-cloud-bigquery
+    #   google-cloud-bigquery-storage
+    #   googleapis-common-protos
+    #   grpcio-status
+    #   proto-plus
 psycopg2-binary==2.9.6
     # via apache-superset
 ptyprocess==0.7.0
@@ -86,24 +212,53 @@ pure-sasl==0.6.2
     # via
     #   pyhive
     #   thrift-sasl
+pydata-google-auth==1.7.0
+    # via pandas-gbq
 pydruid==0.6.5
     # via apache-superset
+pyee==11.0.1
+    # via playwright
+pyfakefs==5.2.2
+    # via -r requirements/development.in
 pyhive[hive_pure_sasl]==0.7.0
     # via apache-superset
 pyinstrument==4.4.0
     # via -r requirements/development.in
 pylint==2.17.7
     # via -r requirements/development.in
+pyproject-api==1.5.2
+    # via tox
+pyproject-hooks==1.0.0
+    # via build
+pytest==7.3.1
+    # via
+    #   -r requirements/development.in
+    #   pytest-cov
+    #   pytest-mock
+pytest-cov==4.0.0
+    # via -r requirements/development.in
+pytest-mock==3.10.0
+    # via -r requirements/development.in
 python-ldap==3.4.3
     # via -r requirements/development.in
+pytz-deprecation-shim==0.1.0.post0
+    # via tzlocal
+requests-oauthlib==1.3.1
+    # via google-auth-oauthlib
+rfc3339-validator==0.1.4
+    # via openapi-schema-validator
 rfc3986==2.0.0
     # via tableschema
 s3transfer==0.6.1
     # via boto3
+sqlalchemy-bigquery==1.6.1
+    # via apache-superset
 sqloxide==0.1.33
     # via -r requirements/development.in
 stack-data==0.6.2
     # via ipython
+statsd==4.0.1
+    # via -r requirements/development.in
 tableschema==1.20.2
     # via apache-superset
 tabulator==1.53.5
@@ -116,19 +271,52 @@ thrift==0.16.0
 thrift-sasl==0.4.3
     # via pyhive
 tomli==2.0.1
-    # via pylint
+    # via
+    #   build
+    #   coverage
+    #   pip-tools
+    #   pylint
+    #   pyproject-api
+    #   pyproject-hooks
+    #   pytest
+    #   tox
 tomlkit==0.11.8
     # via pylint
+toposort==1.10
+    # via pip-compile-multi
+tox==4.6.4
+    # via -r requirements/development.in
+tqdm==4.65.0
+    # via
+    #   cmdstanpy
+    #   prophet
 traitlets==5.9.0
     # via
     #   ipython
     #   matplotlib-inline
+trino==0.328.0
+    # via apache-superset
+tzlocal==4.3
+    # via trino
 unicodecsv==0.14.1
     # via
     #   tableschema
     #   tabulator
+virtualenv==20.23.1
+    # via
+    #   pre-commit
+    #   tox
+websocket-client==1.5.1
+    # via docker
+wheel==0.40.0
+    # via pip-tools
 xlrd==2.0.1
     # via tabulator
+zope-event==4.5.0
+    # via gevent
+zope-interface==5.4.0
+    # via gevent
 
 # The following packages are considered to be unsafe in a requirements file:
+# pip
 # setuptools
diff --git a/requirements/docker.in b/requirements/docker.in
deleted file mode 100644
index 5b65cc68718f3..0000000000000
--- a/requirements/docker.in
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
--r base.in
--e .[postgres,gevent]
-greenlet>=2.0.2
diff --git a/requirements/docker.txt b/requirements/docker.txt
deleted file mode 100644
index 27c135e04c757..0000000000000
--- a/requirements/docker.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-# SHA1:f00a57c70a52607d638c19f64f426f887382927e
-#
-# This file is autogenerated by pip-compile-multi
-# To update, run:
-#
-#    pip-compile-multi
-#
--r base.txt
--e file:.
-    # via
-    #   -r requirements/base.in
-    #   -r requirements/docker.in
-gevent==23.9.1
-    # via apache-superset
-psycopg2-binary==2.9.6
-    # via apache-superset
-zope-event==4.5.0
-    # via gevent
-zope-interface==5.4.0
-    # via gevent
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
diff --git a/requirements/integration.in b/requirements/integration.in
deleted file mode 100644
index 9601b3a0d23bf..0000000000000
--- a/requirements/integration.in
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-pip-compile-multi
-pre-commit
-tox
diff --git a/requirements/integration.txt b/requirements/integration.txt
deleted file mode 100644
index ce55c02b8b2e5..0000000000000
--- a/requirements/integration.txt
+++ /dev/null
@@ -1,74 +0,0 @@
-# SHA1:39179f2c476f94362aa0705be059a488d7e38b6d
-#
-# This file is autogenerated by pip-compile-multi
-# To update, run:
-#
-#    pip-compile-multi
-#
-build==0.10.0
-    # via pip-tools
-cachetools==5.3.2
-    # via tox
-cfgv==3.3.1
-    # via pre-commit
-chardet==5.1.0
-    # via tox
-click==8.1.3
-    # via
-    #   pip-compile-multi
-    #   pip-tools
-colorama==0.4.6
-    # via tox
-distlib==0.3.6
-    # via virtualenv
-filelock==3.12.2
-    # via
-    #   tox
-    #   virtualenv
-identify==2.5.24
-    # via pre-commit
-nodeenv==1.7.0
-    # via pre-commit
-packaging==23.1
-    # via
-    #   build
-    #   pyproject-api
-    #   tox
-pip-compile-multi==2.6.3
-    # via -r requirements/integration.in
-pip-tools==7.3.0
-    # via pip-compile-multi
-platformdirs==3.8.1
-    # via
-    #   tox
-    #   virtualenv
-pluggy==1.2.0
-    # via tox
-pre-commit==3.3.3
-    # via -r requirements/integration.in
-pyproject-api==1.5.2
-    # via tox
-pyproject-hooks==1.0.0
-    # via build
-pyyaml==6.0.1
-    # via pre-commit
-tomli==2.0.1
-    # via
-    #   build
-    #   pip-tools
-    #   pyproject-api
-    #   tox
-toposort==1.10
-    # via pip-compile-multi
-tox==4.6.4
-    # via -r requirements/integration.in
-virtualenv==20.23.1
-    # via
-    #   pre-commit
-    #   tox
-wheel==0.40.0
-    # via pip-tools
-
-# The following packages are considered to be unsafe in a requirements file:
-# pip
-# setuptools
diff --git a/requirements/local.in b/requirements/local.in
deleted file mode 100644
index bc67aed10901f..0000000000000
--- a/requirements/local.in
+++ /dev/null
@@ -1,17 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
--r development.in
diff --git a/requirements/local.txt b/requirements/local.txt
deleted file mode 100644
index c4bd3cd599b36..0000000000000
--- a/requirements/local.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-# SHA1:a71c19ba0170092851941268a0a3dc233feba06d
-#
-# This file is autogenerated by pip-compile-multi
-# To update, run:
-#
-#    pip-compile-multi
-#
--r development.txt
--e file:.
-    # via
-    #   -r requirements/base.in
-    #   -r requirements/development.in
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
diff --git a/requirements/testing.in b/requirements/testing.in
deleted file mode 100644
index 3245886ec854d..0000000000000
--- a/requirements/testing.in
+++ /dev/null
@@ -1,31 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
--r development.in
--r integration.in
--e file:.[bigquery,hive,presto,prophet,trino,gsheets,playwright]
-docker
-flask-testing
-freezegun
-grpcio>=1.55.3
-openapi-spec-validator
-parameterized
-pyfakefs
-pylint
-pytest
-pytest-cov
-pytest-mock
-statsd
diff --git a/requirements/testing.txt b/requirements/testing.txt
deleted file mode 100644
index 55e03e44706da..0000000000000
--- a/requirements/testing.txt
+++ /dev/null
@@ -1,147 +0,0 @@
-# SHA1:a37a1037f359c1101162ef43288178fbf00c487d
-#
-# This file is autogenerated by pip-compile-multi
-# To update, run:
-#
-#    pip-compile-multi
-#
--r development.txt
--r integration.txt
--e file:.
-    # via
-    #   -r requirements/base.in
-    #   -r requirements/development.in
-    #   -r requirements/testing.in
-cmdstanpy==1.1.0
-    # via prophet
-contourpy==1.0.7
-    # via matplotlib
-coverage[toml]==7.2.5
-    # via pytest-cov
-cycler==0.11.0
-    # via matplotlib
-db-dtypes==1.1.1
-    # via pandas-gbq
-docker==6.1.1
-    # via -r requirements/testing.in
-flask-testing==0.8.1
-    # via -r requirements/testing.in
-fonttools==4.43.0
-    # via matplotlib
-freezegun==1.2.2
-    # via -r requirements/testing.in
-google-api-core[grpc]==2.11.0
-    # via
-    #   google-cloud-bigquery
-    #   google-cloud-bigquery-storage
-    #   google-cloud-core
-    #   pandas-gbq
-    #   sqlalchemy-bigquery
-google-auth-oauthlib==1.0.0
-    # via
-    #   pandas-gbq
-    #   pydata-google-auth
-google-cloud-bigquery==3.10.0
-    # via
-    #   apache-superset
-    #   pandas-gbq
-    #   sqlalchemy-bigquery
-google-cloud-bigquery-storage==2.19.1
-    # via
-    #   pandas-gbq
-    #   sqlalchemy-bigquery
-google-cloud-core==2.3.2
-    # via google-cloud-bigquery
-google-crc32c==1.5.0
-    # via google-resumable-media
-google-resumable-media==2.5.0
-    # via google-cloud-bigquery
-googleapis-common-protos==1.59.0
-    # via
-    #   google-api-core
-    #   grpcio-status
-grpcio==1.60.1
-    # via
-    #   -r requirements/testing.in
-    #   google-api-core
-    #   google-cloud-bigquery
-    #   grpcio-status
-grpcio-status==1.60.1
-    # via google-api-core
-iniconfig==2.0.0
-    # via pytest
-jsonschema-spec==0.1.4
-    # via openapi-spec-validator
-kiwisolver==1.4.4
-    # via matplotlib
-matplotlib==3.7.1
-    # via prophet
-oauthlib==3.2.2
-    # via requests-oauthlib
-openapi-schema-validator==0.4.4
-    # via openapi-spec-validator
-openapi-spec-validator==0.5.6
-    # via -r requirements/testing.in
-pandas-gbq==0.19.1
-    # via apache-superset
-parameterized==0.9.0
-    # via -r requirements/testing.in
-pathable==0.4.3
-    # via jsonschema-spec
-playwright==1.41.2
-    # via apache-superset
-prophet==1.1.5
-    # via apache-superset
-proto-plus==1.22.2
-    # via
-    #   google-cloud-bigquery
-    #   google-cloud-bigquery-storage
-protobuf==4.23.0
-    # via
-    #   google-api-core
-    #   google-cloud-bigquery
-    #   google-cloud-bigquery-storage
-    #   googleapis-common-protos
-    #   grpcio-status
-    #   proto-plus
-pydata-google-auth==1.7.0
-    # via pandas-gbq
-pyee==11.0.1
-    # via playwright
-pyfakefs==5.2.2
-    # via -r requirements/testing.in
-pyhive[presto]==0.7.0
-    # via apache-superset
-pytest==7.3.1
-    # via
-    #   -r requirements/testing.in
-    #   pytest-cov
-    #   pytest-mock
-pytest-cov==4.0.0
-    # via -r requirements/testing.in
-pytest-mock==3.10.0
-    # via -r requirements/testing.in
-pytz-deprecation-shim==0.1.0.post0
-    # via tzlocal
-requests-oauthlib==1.3.1
-    # via google-auth-oauthlib
-rfc3339-validator==0.1.4
-    # via openapi-schema-validator
-sqlalchemy-bigquery==1.6.1
-    # via apache-superset
-statsd==4.0.1
-    # via -r requirements/testing.in
-tqdm==4.65.0
-    # via
-    #   cmdstanpy
-    #   prophet
-trino==0.328.0
-    # via apache-superset
-tzlocal==4.3
-    # via trino
-websocket-client==1.5.1
-    # via docker
-
-# The following packages are considered to be unsafe in a requirements file:
-# pip
-# setuptools
diff --git a/tox.ini b/tox.ini
index 1218fb79a305f..28102e01438a9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -27,7 +27,7 @@ commands =
     # no args or tests/* can be passed as an argument to run all tests
     pytest -s {posargs}
 deps =
-    -rrequirements/testing.txt
+    -rrequirements/development.txt
 setenv =
     PYTHONPATH = {toxinidir}
     SUPERSET_TESTENV = true
@@ -147,20 +147,20 @@ deps =
 commands =
     pre-commit run --all-files
 deps =
-    -rrequirements/integration.txt
+    -rrequirements/development.txt
 skip_install = true
 
 [testenv:pylint]
 commands =
     pylint superset
 deps =
-    -rrequirements/testing.txt
+    -rrequirements/development.txt
 
 [testenv:thumbnails]
 setenv =
     SUPERSET_CONFIG = tests.integration_tests.superset_test_config_thumbnails
 deps =
-    -rrequirements/testing.txt
+    -rrequirements/development.txt
 
 [tox]
 envlist =