diff --git a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml index e58e0d4436f8..5c83cec1856e 100644 --- a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml @@ -40,7 +40,6 @@ body: - apache-pig - apache-pinot - apache-spark - - apache-sqoop - apprise - arangodb - asana @@ -88,7 +87,6 @@ body: - papermill - pgvector - pinecone - - plexus - postgres - presto - redis diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 25633c0995d8..c015e457bdf1 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -670,18 +670,18 @@ This is the full list of those extras: .. START EXTRAS HERE aiobotocore, airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill, apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kafka, -apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, -apprise, arangodb, asana, async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups, -cloudant, cncf.kubernetes, cohere, common.io, common.sql, crypto, databricks, datadog, dbt.cloud, +apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.webhdfs, apprise, +arangodb, asana, async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups, cloudant, +cncf.kubernetes, cohere, common.io, common.sql, crypto, databricks, datadog, dbt.cloud, deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, doc_gen, docker, druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, kerberos, kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openai, openfaas, openlineage, opensearch, opsgenie, oracle, otel, pagerduty, -pandas, papermill, password, pgvector, pinecone, pinot, plexus, postgres, presto, rabbitmq, redis, -s3, s3fs, salesforce, samba, saml, segment, sendgrid, sentry, sftp, singularity, slack, smtp, -snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv, -weaviate, webhdfs, winrm, yandex, zendesk +pandas, papermill, password, pgvector, pinecone, pinot, postgres, presto, rabbitmq, redis, s3, s3fs, +salesforce, samba, saml, segment, sendgrid, sentry, sftp, singularity, slack, smtp, snowflake, +spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv, weaviate, +webhdfs, winrm, yandex, zendesk .. END EXTRAS HERE Provider packages diff --git a/INSTALL b/INSTALL index e784b2d057aa..9630c5098523 100644 --- a/INSTALL +++ b/INSTALL @@ -96,18 +96,18 @@ The list of available extras: # START EXTRAS HERE aiobotocore, airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill, apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kafka, -apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, -apprise, arangodb, asana, async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups, -cloudant, cncf.kubernetes, cohere, common.io, common.sql, crypto, databricks, datadog, dbt.cloud, +apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.webhdfs, apprise, +arangodb, asana, async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups, cloudant, +cncf.kubernetes, cohere, common.io, common.sql, crypto, databricks, datadog, dbt.cloud, deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, doc_gen, docker, druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, kerberos, kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openai, openfaas, openlineage, opensearch, opsgenie, oracle, otel, pagerduty, -pandas, papermill, password, pgvector, pinecone, pinot, plexus, postgres, presto, rabbitmq, redis, -s3, s3fs, salesforce, samba, saml, segment, sendgrid, sentry, sftp, singularity, slack, smtp, -snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv, -weaviate, webhdfs, winrm, yandex, zendesk +pandas, papermill, password, pgvector, pinecone, pinot, postgres, presto, rabbitmq, redis, s3, s3fs, +salesforce, samba, saml, segment, sendgrid, sentry, sftp, singularity, slack, smtp, snowflake, +spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv, weaviate, +webhdfs, winrm, yandex, zendesk # END EXTRAS HERE # For installing Airflow in development environments - see CONTRIBUTING.rst diff --git a/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst b/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst index 28a7587827b3..27ae7ef4564b 100644 --- a/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst +++ b/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst @@ -514,14 +514,15 @@ providers (and then remove all the code and documentation related to the provide The "removed: true" flag will cause the provider to be available for the following commands (note that such provider has to be explicitly added as selected to the package - such provider will not be included in -the available list of providers): +the available list of providers or when documentation is built unless --include-removed-providers +flag is used): * ``breeze build-docs`` * ``breeze release-management prepare-provider-documentation`` * ``breeze release-management prepare-provider-packages`` * ``breeze release-management publish-docs`` -For all those commands, release manager needs to specify such to-be-removed provider explicitly as extra -command during the release process. Except the changelog that needs to be maintained manually, all other -documentation (main page of the provider documentation, PyPI README), will be automatically updated -to include removal notice. +For all those commands, release manager needs to specify ``--include-removed-providers`` when all providers +are built or must add the provider id explicitly during the release process. +Except the changelog that needs to be maintained manually, all other documentation (main page of the provider +documentation, PyPI README), will be automatically updated to include removal notice. diff --git a/airflow/providers/apache/sqoop/provider.yaml b/airflow/providers/apache/sqoop/provider.yaml index d7482170fe37..d6bc9b6c6070 100644 --- a/airflow/providers/apache/sqoop/provider.yaml +++ b/airflow/providers/apache/sqoop/provider.yaml @@ -21,9 +21,11 @@ name: Apache Sqoop description: | `Apache Sqoop `__ -suspended: false -source-date-epoch: 1700827444 +removed: true +suspended: true +source-date-epoch: 1701983361 versions: + - 4.2.0 - 4.1.0 - 4.0.0 - 3.2.1 diff --git a/airflow/providers/plexus/provider.yaml b/airflow/providers/plexus/provider.yaml index 29d8e4d24c74..64752b5fd53c 100644 --- a/airflow/providers/plexus/provider.yaml +++ b/airflow/providers/plexus/provider.yaml @@ -21,9 +21,11 @@ name: Plexus description: | `Plexus `__ -suspended: false -source-date-epoch: 1700827488 +removed: true +suspended: true +source-date-epoch: 1701983408 versions: + - 3.4.0 - 3.3.0 - 3.2.2 - 3.2.1 diff --git a/dev/breeze/tests/test_packages.py b/dev/breeze/tests/test_packages.py index bb89dbf6b00f..f50b8e2c349b 100644 --- a/dev/breeze/tests/test_packages.py +++ b/dev/breeze/tests/test_packages.py @@ -109,17 +109,17 @@ def test_get_provider_requirements(): def test_get_removed_providers(): # Modify it every time we schedule provider for removal or remove it - assert ["daskexecutor"] == get_removed_provider_ids() + assert ["apache.sqoop", "daskexecutor", "plexus"] == get_removed_provider_ids() def test_get_suspended_provider_ids(): # Modify it every time we suspend/resume provider - assert ["daskexecutor"] == get_suspended_provider_ids() + assert ["apache.sqoop", "daskexecutor", "plexus"] == get_suspended_provider_ids() def test_get_suspended_provider_folders(): # Modify it every time we suspend/resume provider - assert ["daskexecutor"] == get_suspended_provider_folders() + assert ["apache/sqoop", "daskexecutor", "plexus"] == get_suspended_provider_folders() @pytest.mark.parametrize( diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst index 80c285b5aab9..324b5e302309 100644 --- a/docs/apache-airflow/extra-packages-ref.rst +++ b/docs/apache-airflow/extra-packages-ref.rst @@ -144,8 +144,6 @@ custom bash/python providers). +---------------------+-----------------------------------------------------+------------------------------------------------+ | apache.spark | ``pip install 'apache-airflow[apache.spark]'`` | All Spark related operators & hooks | +---------------------+-----------------------------------------------------+------------------------------------------------+ -| apache.sqoop | ``pip install 'apache-airflow[apache.sqoop]'`` | All Sqoop related operators & hooks | -+---------------------+-----------------------------------------------------+------------------------------------------------+ | apache.webhdfs | ``pip install 'apache-airflow[apache.webhdfs]'`` | HDFS hooks and operators | +---------------------+-----------------------------------------------------+------------------------------------------------+ @@ -204,8 +202,6 @@ These are extras that add dependencies needed for integration with external serv +---------------------+-----------------------------------------------------+-----------------------------------------------------+ | pinecone | ``pip install 'apache-airflow[pinecone]'`` | Pinecone Operators and Hooks | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ -| plexus | ``pip install 'apache-airflow[plexus]'`` | Plexus service of CoreScientific.com AI platform | -+---------------------+-----------------------------------------------------+-----------------------------------------------------+ | salesforce | ``pip install 'apache-airflow[salesforce]'`` | Salesforce hook | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ | sendgrid | ``pip install 'apache-airflow[sendgrid]'`` | Send email using sendgrid | diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index ef413f15f9f0..8d2f31048b6e 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -198,13 +198,6 @@ ], "excluded-python-versions": [] }, - "apache.sqoop": { - "deps": [ - "apache-airflow>=2.6.0" - ], - "cross-providers-deps": [], - "excluded-python-versions": [] - }, "apprise": { "deps": [ "apache-airflow>=2.6.0", @@ -751,14 +744,6 @@ "cross-providers-deps": [], "excluded-python-versions": [] }, - "plexus": { - "deps": [ - "apache-airflow>=2.6.0", - "arrow>=0.16.0" - ], - "cross-providers-deps": [], - "excluded-python-versions": [] - }, "postgres": { "deps": [ "apache-airflow-providers-common-sql>=1.3.1", diff --git a/images/breeze/output_release-management_generate-issue-content-providers.svg b/images/breeze/output_release-management_generate-issue-content-providers.svg index 0ed99d0f35eb..46fce9329be1 100644 --- a/images/breeze/output_release-management_generate-issue-content-providers.svg +++ b/images/breeze/output_release-management_generate-issue-content-providers.svg @@ -143,13 +143,13 @@ breeze release-management generate-issue-content-providers                                                             [OPTIONS] [airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink |  apache.hdfs | apache.hive | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot |    -apache.spark | apache.sqoop | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes |      -cohere | common.io | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch |     -exasol | facebook | ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins |               -microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         -openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | plexus |       -postgres | presto | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |  -sqlite | ssh | tableau | tabular | telegram | trino | vertica | weaviate | yandex | zendesk]...                        +apache.spark | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io +| common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol | facebook |    +ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql +| microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch |     +opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | redis | salesforce | samba |     +segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | tableau | tabular | telegram |     +trino | vertica | weaviate | yandex | zendesk]...                                                                      Generates content for issue to test the release. diff --git a/images/breeze/output_release-management_generate-issue-content-providers.txt b/images/breeze/output_release-management_generate-issue-content-providers.txt index 8c14235124a3..896b36c77ab2 100644 --- a/images/breeze/output_release-management_generate-issue-content-providers.txt +++ b/images/breeze/output_release-management_generate-issue-content-providers.txt @@ -1 +1 @@ -a01f887299d575afcd64b5e1dd537628 +7361238411e5097d70e0e359b5246a2a diff --git a/images/breeze/output_release-management_prepare-provider-documentation.svg b/images/breeze/output_release-management_prepare-provider-documentation.svg index a4a10ee97f55..21e781615d87 100644 --- a/images/breeze/output_release-management_prepare-provider-documentation.svg +++ b/images/breeze/output_release-management_prepare-provider-documentation.svg @@ -173,13 +173,13 @@ breeze release-management prepare-provider-documentation                                                               [OPTIONS] [airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink |  apache.hdfs | apache.hive | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot |    -apache.spark | apache.sqoop | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes |      -cohere | common.io | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch |     -exasol | facebook | ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins |               -microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         -openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | plexus |       -postgres | presto | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |  -sqlite | ssh | tableau | tabular | telegram | trino | vertica | weaviate | yandex | zendesk]...                        +apache.spark | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io +| common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol | facebook |    +ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql +| microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch |     +opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | redis | salesforce | samba |     +segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | tableau | tabular | telegram |     +trino | vertica | weaviate | yandex | zendesk]...                                                                      Prepare CHANGELOG, README and COMMITS information for providers. diff --git a/images/breeze/output_release-management_prepare-provider-documentation.txt b/images/breeze/output_release-management_prepare-provider-documentation.txt index 667579650486..1cab6ad19775 100644 --- a/images/breeze/output_release-management_prepare-provider-documentation.txt +++ b/images/breeze/output_release-management_prepare-provider-documentation.txt @@ -1 +1 @@ -077e206ba81e0ae33adfc5370d04607f +d73adb90b5bd1ea6c74c2c3ab23f10c6 diff --git a/images/breeze/output_release-management_prepare-provider-packages.svg b/images/breeze/output_release-management_prepare-provider-packages.svg index 11bad0976377..9283feb0acbe 100644 --- a/images/breeze/output_release-management_prepare-provider-packages.svg +++ b/images/breeze/output_release-management_prepare-provider-packages.svg @@ -158,13 +158,13 @@ breeze release-management prepare-provider-packages                                                                    [OPTIONS] [airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink |  apache.hdfs | apache.hive | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot |    -apache.spark | apache.sqoop | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes |      -cohere | common.io | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch |     -exasol | facebook | ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins |               -microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         -openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | plexus |       -postgres | presto | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |  -sqlite | ssh | tableau | tabular | telegram | trino | vertica | weaviate | yandex | zendesk]...                        +apache.spark | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io +| common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol | facebook |    +ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql +| microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch |     +opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | redis | salesforce | samba |     +segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | tableau | tabular | telegram |     +trino | vertica | weaviate | yandex | zendesk]...                                                                      Prepare sdist/whl packages of Airflow Providers. diff --git a/images/breeze/output_release-management_prepare-provider-packages.txt b/images/breeze/output_release-management_prepare-provider-packages.txt index 293f477ed70d..9cc18c5ef97c 100644 --- a/images/breeze/output_release-management_prepare-provider-packages.txt +++ b/images/breeze/output_release-management_prepare-provider-packages.txt @@ -1 +1 @@ -8d69f1d20f6df599cda9320f762fba4d +b2ea32dc47688c8e30624a120c6162b8 diff --git a/images/breeze/output_sbom_generate-providers-requirements.svg b/images/breeze/output_sbom_generate-providers-requirements.svg index 2929a5fd5797..cc6e4a41e6b9 100644 --- a/images/breeze/output_sbom_generate-providers-requirements.svg +++ b/images/breeze/output_sbom_generate-providers-requirements.svg @@ -185,15 +185,15 @@ │--provider-idProvider id to generate the requirements for                                                  │ │(airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid |  │ │apache.flink | apache.hdfs | apache.hive | apache.impala | apache.kafka | apache.kylin |      │ -│apache.livy | apache.pig | apache.pinot | apache.spark | apache.sqoop | apprise | arangodb |  │ -│asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io | common.sql│ -│| databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol |   │ +│apache.livy | apache.pig | apache.pinot | apache.spark | apprise | arangodb | asana |         │ +│atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io | common.sql |      │ +│databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol |     │ │facebook | ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins │ │| microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j│ │| odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty |       │ -│papermill | pgvector | pinecone | plexus | postgres | presto | redis | salesforce | samba |   │ -│segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | tableau | │ -│tabular | telegram | trino | vertica | weaviate | yandex | zendesk)                           │ +│papermill | pgvector | pinecone | postgres | presto | redis | salesforce | samba | segment |  │ +│sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | tableau | tabular | │ +│telegram | trino | vertica | weaviate | yandex | zendesk)                                     │ │--provider-versionProvider version to generate the requirements for i.e `2.1.0`. `latest` is also a supported   │ │value to account for the most recent version of the provider                                  │ │(TEXT)                                                                                        │ diff --git a/images/breeze/output_sbom_generate-providers-requirements.txt b/images/breeze/output_sbom_generate-providers-requirements.txt index f53f468d67e0..f696cff21133 100644 --- a/images/breeze/output_sbom_generate-providers-requirements.txt +++ b/images/breeze/output_sbom_generate-providers-requirements.txt @@ -1 +1 @@ -38c82663d57bbd39a51446325c896312 +2dbd6d5c101420d1a788b3cd9325d892