diff --git a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml
index e58e0d4436f8..5c83cec1856e 100644
--- a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml
@@ -40,7 +40,6 @@ body:
- apache-pig
- apache-pinot
- apache-spark
- - apache-sqoop
- apprise
- arangodb
- asana
@@ -88,7 +87,6 @@ body:
- papermill
- pgvector
- pinecone
- - plexus
- postgres
- presto
- redis
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index 25633c0995d8..c015e457bdf1 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -670,18 +670,18 @@ This is the full list of those extras:
.. START EXTRAS HERE
aiobotocore, airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra,
apache.drill, apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kafka,
-apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs,
-apprise, arangodb, asana, async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups,
-cloudant, cncf.kubernetes, cohere, common.io, common.sql, crypto, databricks, datadog, dbt.cloud,
+apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.webhdfs, apprise,
+arangodb, asana, async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups, cloudant,
+cncf.kubernetes, cohere, common.io, common.sql, crypto, databricks, datadog, dbt.cloud,
deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, doc_gen, docker,
druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google,
google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, kerberos, kubernetes,
ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql,
mysql, neo4j, odbc, openai, openfaas, openlineage, opensearch, opsgenie, oracle, otel, pagerduty,
-pandas, papermill, password, pgvector, pinecone, pinot, plexus, postgres, presto, rabbitmq, redis,
-s3, s3fs, salesforce, samba, saml, segment, sendgrid, sentry, sftp, singularity, slack, smtp,
-snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv,
-weaviate, webhdfs, winrm, yandex, zendesk
+pandas, papermill, password, pgvector, pinecone, pinot, postgres, presto, rabbitmq, redis, s3, s3fs,
+salesforce, samba, saml, segment, sendgrid, sentry, sftp, singularity, slack, smtp, snowflake,
+spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv, weaviate,
+webhdfs, winrm, yandex, zendesk
.. END EXTRAS HERE
Provider packages
diff --git a/INSTALL b/INSTALL
index e784b2d057aa..9630c5098523 100644
--- a/INSTALL
+++ b/INSTALL
@@ -96,18 +96,18 @@ The list of available extras:
# START EXTRAS HERE
aiobotocore, airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra,
apache.drill, apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kafka,
-apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs,
-apprise, arangodb, asana, async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups,
-cloudant, cncf.kubernetes, cohere, common.io, common.sql, crypto, databricks, datadog, dbt.cloud,
+apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.webhdfs, apprise,
+arangodb, asana, async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups, cloudant,
+cncf.kubernetes, cohere, common.io, common.sql, crypto, databricks, datadog, dbt.cloud,
deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, doc_gen, docker,
druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google,
google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, kerberos, kubernetes,
ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql,
mysql, neo4j, odbc, openai, openfaas, openlineage, opensearch, opsgenie, oracle, otel, pagerduty,
-pandas, papermill, password, pgvector, pinecone, pinot, plexus, postgres, presto, rabbitmq, redis,
-s3, s3fs, salesforce, samba, saml, segment, sendgrid, sentry, sftp, singularity, slack, smtp,
-snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv,
-weaviate, webhdfs, winrm, yandex, zendesk
+pandas, papermill, password, pgvector, pinecone, pinot, postgres, presto, rabbitmq, redis, s3, s3fs,
+salesforce, samba, saml, segment, sendgrid, sentry, sftp, singularity, slack, smtp, snowflake,
+spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv, weaviate,
+webhdfs, winrm, yandex, zendesk
# END EXTRAS HERE
# For installing Airflow in development environments - see CONTRIBUTING.rst
diff --git a/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst b/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst
index 28a7587827b3..27ae7ef4564b 100644
--- a/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst
+++ b/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst
@@ -514,14 +514,15 @@ providers (and then remove all the code and documentation related to the provide
The "removed: true" flag will cause the provider to be available for the following commands (note that such
provider has to be explicitly added as selected to the package - such provider will not be included in
-the available list of providers):
+the available list of providers or when documentation is built unless --include-removed-providers
+flag is used):
* ``breeze build-docs``
* ``breeze release-management prepare-provider-documentation``
* ``breeze release-management prepare-provider-packages``
* ``breeze release-management publish-docs``
-For all those commands, release manager needs to specify such to-be-removed provider explicitly as extra
-command during the release process. Except the changelog that needs to be maintained manually, all other
-documentation (main page of the provider documentation, PyPI README), will be automatically updated
-to include removal notice.
+For all those commands, release manager needs to specify ``--include-removed-providers`` when all providers
+are built or must add the provider id explicitly during the release process.
+Except the changelog that needs to be maintained manually, all other documentation (main page of the provider
+documentation, PyPI README), will be automatically updated to include removal notice.
diff --git a/airflow/providers/apache/sqoop/provider.yaml b/airflow/providers/apache/sqoop/provider.yaml
index d7482170fe37..d6bc9b6c6070 100644
--- a/airflow/providers/apache/sqoop/provider.yaml
+++ b/airflow/providers/apache/sqoop/provider.yaml
@@ -21,9 +21,11 @@ name: Apache Sqoop
description: |
`Apache Sqoop `__
-suspended: false
-source-date-epoch: 1700827444
+removed: true
+suspended: true
+source-date-epoch: 1701983361
versions:
+ - 4.2.0
- 4.1.0
- 4.0.0
- 3.2.1
diff --git a/airflow/providers/plexus/provider.yaml b/airflow/providers/plexus/provider.yaml
index 29d8e4d24c74..64752b5fd53c 100644
--- a/airflow/providers/plexus/provider.yaml
+++ b/airflow/providers/plexus/provider.yaml
@@ -21,9 +21,11 @@ name: Plexus
description: |
`Plexus `__
-suspended: false
-source-date-epoch: 1700827488
+removed: true
+suspended: true
+source-date-epoch: 1701983408
versions:
+ - 3.4.0
- 3.3.0
- 3.2.2
- 3.2.1
diff --git a/dev/breeze/tests/test_packages.py b/dev/breeze/tests/test_packages.py
index bb89dbf6b00f..f50b8e2c349b 100644
--- a/dev/breeze/tests/test_packages.py
+++ b/dev/breeze/tests/test_packages.py
@@ -109,17 +109,17 @@ def test_get_provider_requirements():
def test_get_removed_providers():
# Modify it every time we schedule provider for removal or remove it
- assert ["daskexecutor"] == get_removed_provider_ids()
+ assert ["apache.sqoop", "daskexecutor", "plexus"] == get_removed_provider_ids()
def test_get_suspended_provider_ids():
# Modify it every time we suspend/resume provider
- assert ["daskexecutor"] == get_suspended_provider_ids()
+ assert ["apache.sqoop", "daskexecutor", "plexus"] == get_suspended_provider_ids()
def test_get_suspended_provider_folders():
# Modify it every time we suspend/resume provider
- assert ["daskexecutor"] == get_suspended_provider_folders()
+ assert ["apache/sqoop", "daskexecutor", "plexus"] == get_suspended_provider_folders()
@pytest.mark.parametrize(
diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst
index 80c285b5aab9..324b5e302309 100644
--- a/docs/apache-airflow/extra-packages-ref.rst
+++ b/docs/apache-airflow/extra-packages-ref.rst
@@ -144,8 +144,6 @@ custom bash/python providers).
+---------------------+-----------------------------------------------------+------------------------------------------------+
| apache.spark | ``pip install 'apache-airflow[apache.spark]'`` | All Spark related operators & hooks |
+---------------------+-----------------------------------------------------+------------------------------------------------+
-| apache.sqoop | ``pip install 'apache-airflow[apache.sqoop]'`` | All Sqoop related operators & hooks |
-+---------------------+-----------------------------------------------------+------------------------------------------------+
| apache.webhdfs | ``pip install 'apache-airflow[apache.webhdfs]'`` | HDFS hooks and operators |
+---------------------+-----------------------------------------------------+------------------------------------------------+
@@ -204,8 +202,6 @@ These are extras that add dependencies needed for integration with external serv
+---------------------+-----------------------------------------------------+-----------------------------------------------------+
| pinecone | ``pip install 'apache-airflow[pinecone]'`` | Pinecone Operators and Hooks |
+---------------------+-----------------------------------------------------+-----------------------------------------------------+
-| plexus | ``pip install 'apache-airflow[plexus]'`` | Plexus service of CoreScientific.com AI platform |
-+---------------------+-----------------------------------------------------+-----------------------------------------------------+
| salesforce | ``pip install 'apache-airflow[salesforce]'`` | Salesforce hook |
+---------------------+-----------------------------------------------------+-----------------------------------------------------+
| sendgrid | ``pip install 'apache-airflow[sendgrid]'`` | Send email using sendgrid |
diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json
index ef413f15f9f0..8d2f31048b6e 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -198,13 +198,6 @@
],
"excluded-python-versions": []
},
- "apache.sqoop": {
- "deps": [
- "apache-airflow>=2.6.0"
- ],
- "cross-providers-deps": [],
- "excluded-python-versions": []
- },
"apprise": {
"deps": [
"apache-airflow>=2.6.0",
@@ -751,14 +744,6 @@
"cross-providers-deps": [],
"excluded-python-versions": []
},
- "plexus": {
- "deps": [
- "apache-airflow>=2.6.0",
- "arrow>=0.16.0"
- ],
- "cross-providers-deps": [],
- "excluded-python-versions": []
- },
"postgres": {
"deps": [
"apache-airflow-providers-common-sql>=1.3.1",
diff --git a/images/breeze/output_release-management_generate-issue-content-providers.svg b/images/breeze/output_release-management_generate-issue-content-providers.svg
index 0ed99d0f35eb..46fce9329be1 100644
--- a/images/breeze/output_release-management_generate-issue-content-providers.svg
+++ b/images/breeze/output_release-management_generate-issue-content-providers.svg
@@ -143,13 +143,13 @@
breeze release-management generate-issue-content-providers
[OPTIONS] [airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink |
apache.hdfs | apache.hive | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot |
-apache.spark | apache.sqoop | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes |
-cohere | common.io | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch |
-exasol | facebook | ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins |
-microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |
-openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | plexus |
-postgres | presto | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |
-sqlite | ssh | tableau | tabular | telegram | trino | vertica | weaviate | yandex | zendesk]...
+apache.spark | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io
+| common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol | facebook |
+ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql
+| microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch |
+opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | redis | salesforce | samba |
+segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | tableau | tabular | telegram |
+trino | vertica | weaviate | yandex | zendesk]...
Generates content for issue to test the release.
diff --git a/images/breeze/output_release-management_generate-issue-content-providers.txt b/images/breeze/output_release-management_generate-issue-content-providers.txt
index 8c14235124a3..896b36c77ab2 100644
--- a/images/breeze/output_release-management_generate-issue-content-providers.txt
+++ b/images/breeze/output_release-management_generate-issue-content-providers.txt
@@ -1 +1 @@
-a01f887299d575afcd64b5e1dd537628
+7361238411e5097d70e0e359b5246a2a
diff --git a/images/breeze/output_release-management_prepare-provider-documentation.svg b/images/breeze/output_release-management_prepare-provider-documentation.svg
index a4a10ee97f55..21e781615d87 100644
--- a/images/breeze/output_release-management_prepare-provider-documentation.svg
+++ b/images/breeze/output_release-management_prepare-provider-documentation.svg
@@ -173,13 +173,13 @@
breeze release-management prepare-provider-documentation
[OPTIONS] [airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink |
apache.hdfs | apache.hive | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot |
-apache.spark | apache.sqoop | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes |
-cohere | common.io | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch |
-exasol | facebook | ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins |
-microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |
-openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | plexus |
-postgres | presto | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |
-sqlite | ssh | tableau | tabular | telegram | trino | vertica | weaviate | yandex | zendesk]...
+apache.spark | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io
+| common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol | facebook |
+ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql
+| microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch |
+opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | redis | salesforce | samba |
+segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | tableau | tabular | telegram |
+trino | vertica | weaviate | yandex | zendesk]...
Prepare CHANGELOG, README and COMMITS information for providers.
diff --git a/images/breeze/output_release-management_prepare-provider-documentation.txt b/images/breeze/output_release-management_prepare-provider-documentation.txt
index 667579650486..1cab6ad19775 100644
--- a/images/breeze/output_release-management_prepare-provider-documentation.txt
+++ b/images/breeze/output_release-management_prepare-provider-documentation.txt
@@ -1 +1 @@
-077e206ba81e0ae33adfc5370d04607f
+d73adb90b5bd1ea6c74c2c3ab23f10c6
diff --git a/images/breeze/output_release-management_prepare-provider-packages.svg b/images/breeze/output_release-management_prepare-provider-packages.svg
index 11bad0976377..9283feb0acbe 100644
--- a/images/breeze/output_release-management_prepare-provider-packages.svg
+++ b/images/breeze/output_release-management_prepare-provider-packages.svg
@@ -158,13 +158,13 @@
breeze release-management prepare-provider-packages
[OPTIONS] [airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink |
apache.hdfs | apache.hive | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot |
-apache.spark | apache.sqoop | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes |
-cohere | common.io | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch |
-exasol | facebook | ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins |
-microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |
-openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | plexus |
-postgres | presto | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |
-sqlite | ssh | tableau | tabular | telegram | trino | vertica | weaviate | yandex | zendesk]...
+apache.spark | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io
+| common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol | facebook |
+ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql
+| microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch |
+opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | redis | salesforce | samba |
+segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | tableau | tabular | telegram |
+trino | vertica | weaviate | yandex | zendesk]...
Prepare sdist/whl packages of Airflow Providers.
diff --git a/images/breeze/output_release-management_prepare-provider-packages.txt b/images/breeze/output_release-management_prepare-provider-packages.txt
index 293f477ed70d..9cc18c5ef97c 100644
--- a/images/breeze/output_release-management_prepare-provider-packages.txt
+++ b/images/breeze/output_release-management_prepare-provider-packages.txt
@@ -1 +1 @@
-8d69f1d20f6df599cda9320f762fba4d
+b2ea32dc47688c8e30624a120c6162b8
diff --git a/images/breeze/output_sbom_generate-providers-requirements.svg b/images/breeze/output_sbom_generate-providers-requirements.svg
index 2929a5fd5797..cc6e4a41e6b9 100644
--- a/images/breeze/output_sbom_generate-providers-requirements.svg
+++ b/images/breeze/output_sbom_generate-providers-requirements.svg
@@ -185,15 +185,15 @@
│--provider-idProvider id to generate the requirements for │
│(airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid | │
│apache.flink | apache.hdfs | apache.hive | apache.impala | apache.kafka | apache.kylin | │
-│apache.livy | apache.pig | apache.pinot | apache.spark | apache.sqoop | apprise | arangodb | │
-│asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io | common.sql│
-│| databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol | │
+│apache.livy | apache.pig | apache.pinot | apache.spark | apprise | arangodb | asana | │
+│atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io | common.sql | │
+│databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol | │
│facebook | ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins │
│| microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j│
│| odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | │
-│papermill | pgvector | pinecone | plexus | postgres | presto | redis | salesforce | samba | │
-│segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | tableau | │
-│tabular | telegram | trino | vertica | weaviate | yandex | zendesk) │
+│papermill | pgvector | pinecone | postgres | presto | redis | salesforce | samba | segment | │
+│sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | tableau | tabular | │
+│telegram | trino | vertica | weaviate | yandex | zendesk) │
│--provider-versionProvider version to generate the requirements for i.e `2.1.0`. `latest` is also a supported │
│value to account for the most recent version of the provider │
│(TEXT) │
diff --git a/images/breeze/output_sbom_generate-providers-requirements.txt b/images/breeze/output_sbom_generate-providers-requirements.txt
index f53f468d67e0..f696cff21133 100644
--- a/images/breeze/output_sbom_generate-providers-requirements.txt
+++ b/images/breeze/output_sbom_generate-providers-requirements.txt
@@ -1 +1 @@
-38c82663d57bbd39a51446325c896312
+2dbd6d5c101420d1a788b3cd9325d892