Merge branch 'main' into fix/mapped_op_templating

.asf.yaml

@@ -69,17 +69,20 @@ github:
     v2-6-stable:
       required_pull_request_reviews:
         required_approving_review_count: 1
+    v2-7-stable:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
 
   collaborators:
     - mhenc
-    - ferruzzi
     - pankajkoti
     - utkarsharma2
     - Lee-W
     - sunank200
     - amoghrajesh
     - jens-scheffler-bosch
     - nathadfield
+    - RNHTTR
 
 notifications:
   jobs: [email protected]

.dockerignore

@@ -42,6 +42,8 @@
 # Add tests and kubernetes_tests to context.
 !tests
 !kubernetes_tests
+!helm_tests
+!docker_tests
 
 !.coveragerc
 !.rat-excludes

.github/CODEOWNERS

@@ -11,7 +11,6 @@
 
 # Kubernetes
 /airflow/kubernetes/ @dstandish @jedcunningham
-/airflow/kubernetes_executor_templates/ @dstandish @jedcunningham
 /airflow/executors/celery_kubernetes_executor.py @dstandish @jedcunningham
 /airflow/executors/kubernetes_executor.py @dstandish @jedcunningham
 

.github/ISSUE_TEMPLATE/airflow_bug_report.yml

@@ -25,7 +25,8 @@ body:
         the latest release or main to see if the issue is fixed before reporting it.
       multiple: false
       options:
-        - "2.6.2"
+        - "2.7.0b1"
+        - "2.6.3"
         - "main (development)"
         - "Other Airflow 2 version (please specify below)"
     validations:

.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml

@@ -49,6 +49,7 @@ body:
         - cloudant
         - cncf-kubernetes
         - common-sql
+        - daskexecutor
         - databricks
         - datadog
         - dbt-cloud

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,3 +1,22 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
 <!--
 Thank you for contributing! Please make sure that your code changes
 are covered with tests. And in case of new features or big changes
@@ -14,9 +33,11 @@ How to write a good git commit message:
 http://chris.beams.io/posts/git-commit/
 -->
 
+
+
+<!-- Please keep an empty line above the dashes. -->
 ---
 **^ Add meaningful description above**
-
 Read the **[Pull Request Guidelines](https://github.com/apache/airflow/blob/main/CONTRIBUTING.rst#pull-request-guidelines)** for more information.
 In case of fundamental code changes, an Airflow Improvement Proposal ([AIP](https://cwiki.apache.org/confluence/display/AIRFLOW/Airflow+Improvement+Proposals)) is needed.
 In case of a new dependency, check compliance with the [ASF 3rd Party License Policy](https://www.apache.org/legal/resolved.html#category-x).

.github/SECURITY.md

@@ -0,0 +1,82 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+This document contains information on how to report security vulnerabilities in Apache Airflow and
+how the security issues reported to Apache Airflow security team are handled. If you would like
+to learn about the security model of Airflow head to
+[Airflow Security](https://airflow.apache.org/docs/apache-airflow/stable/security/)
+
+## Reporting Vulnerabilities
+
+**⚠️ Please do not file GitHub issues for security vulnerabilities as they are public! ⚠️**
+
+The Apache Software Foundation takes security issues very seriously. Apache
+Airflow specifically offers security features and is responsive to issues
+around its features. If you have any concern around Airflow Security or believe
+you have uncovered a vulnerability, we suggest that you get in touch via the
+e-mail address [[email protected]](mailto:[email protected]).
+
+**Only** use the security e-mail address to report undisclosed security vulnerabilities in Apache
+Airflow and to manage the process of fixing such vulnerabilities. We do not accept regular
+bug reports or other security-related queries at this address. We will ignore mail
+sent to this address that does not relate to an undisclosed security problem
+in the Apache Airflow project. Please follow regular communication channels described in
+the [Airflow Community](https://airflow.apache.org/community/index.html) page for
+inquiries, questions and other discussions related to the process or issues.
+
+Specifically, we will ignore results of security scans that contain a list of
+dependencies of Airflow with dependencies in Airflow Docker reference image - there
+is a page that describes how the
+[Airflow reference Image is fixed at release time](https://airflow.apache.org/docs/docker-stack/index.html#fixing-images-at-release-time)
+and providing helpful instructions explaining how you can build your
+own image and manage dependencies of Airflow in your own image.
+
+Please send one plain-text email for each vulnerability you are reporting including an explanation
+of how it affects Airflow security. We may ask that you resubmit your report if you send it as an image,
+movie, HTML, or PDF attachment when you could as easily describe it with plain text.
+
+Before reporting vulnerabilities, please make sure to read and understand the
+[security model](https://airflow.apache.org/docs/apache-airflow/stable/security/) of Airflow, because
+some of the potential security vulnerabilities that are valid for projects that are publicly accessible
+from the Internet, are not valid for Airflow. Airflow is not designed to be used by untrusted users, and some
+trusted users are trusted enough to do a variety of operations that could be considered as vulnerabilities
+in other products/circumstances. Therefore, some potential security vulnerabilities do not
+apply to Airflow, or have a different severity than some generic scoring systems (for example `CVSS`)
+calculation suggests.
+
+The [Airflow Security Team](https://github.com/apache/airflow/blob/main/CONTRIBUTING.rst#security-team) will get back to you after assessing the report.
+
+If you wish to know more about the ASF security process, the
+[ASF Security team's page](https://www.apache.org/security/) describes
+how vulnerability reports are handled in general by all ASF projects, and includes PGP keys if
+you wish to use them when you report the issues.
+
+## Security vulnerabilities in Airflow and Airflow community managed providers
+
+Airflow core package is released separately from provider packages. While Airflow comes with ``constraints``
+which describe which version of providers have been tested when the version of Airflow was released, the
+users of Airflow are advised to install providers independently from Airflow core when they want to apply
+security fixes found and released in providers. Therefore, the issues found and fixed in providers do
+not apply to the Airflow core package. There are also Airflow providers released by 3rd-parties, but the
+Airflow community is not responsible for releasing and announcing security vulnerabilities in them, this
+is handled entirely by the 3rd-parties that release their own providers.
+
+## How are security issues handled in Airflow
+
+Security issues in Airflow are handled by the Airflow Security Team. Details about the Airflow Security Team and how members of it are chosen can be found in the [Contributing documentation](https://github.com/apache/airflow/blob/main/CONTRIBUTING.rst#security-team).

.github/actions/build-ci-images/action.yml

@@ -54,6 +54,7 @@ runs:
       run: breeze ci-image build --push --empty-image --run-in-parallel
       env:
         IMAGE_TAG: ${{ env.IMAGE_TAG }}
+        COMMIT_SHA: ${{ github.sha }}
     - name: "Fix ownership"
       shell: bash
       run: breeze ci fix-ownership

.github/actions/build-prod-images/action.yml

@@ -61,6 +61,8 @@ runs:
       run: >
         breeze prod-image build --tag-as-latest --run-in-parallel --push
         --install-packages-from-context --upgrade-on-failure
+      env:
+        COMMIT_SHA: ${{ github.sha }}
     - name: Push empty PROD images ${{ env.IMAGE_TAG }}
       shell: bash
       run: breeze prod-image build --cleanup-context --push --empty-image --run-in-parallel

.github/actions/migration_tests/action.yml

@@ -25,13 +25,32 @@ runs:
       shell: bash
       run: >
         breeze shell "airflow db reset --skip-init -y &&
-        airflow db upgrade --to-revision heads &&
+        airflow db migrate --to-revision heads &&
         airflow db downgrade -r e959f08ac86c -y &&
-        airflow db upgrade"
+        airflow db migrate"
+      env:
+        COMPOSE_PROJECT_NAME: "docker-compose"
+    - name: "Bring composer down"
+      shell: bash
+      run: breeze down
+      env:
+        COMPOSE_PROJECT_NAME: "docker-compose"
     - name: "Test downgrade ORM ${{env.BACKEND}}"
       shell: bash
       run: >
         breeze shell "airflow db reset -y &&
-        airflow db upgrade &&
+        airflow db migrate &&
         airflow db downgrade -r e959f08ac86c -y &&
-        airflow db upgrade"
+        airflow db migrate"
+        COMPOSE_PROJECT_NAME: "docker-compose"
+      env:
+        COMPOSE_PROJECT_NAME: "docker-compose"
+    - name: "Bring any containers left down"
+      shell: bash
+      run: breeze down
+      env:
+        COMPOSE_PROJECT_NAME: "docker-compose"
+    - name: "Dump logs on failure ${{env.BACKEND}}"
+      shell: bash
+      run: docker ps -q | xargs docker logs
+      if: failure()