diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index a25ad735adada..0a81cd83947eb 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -2111,6 +2111,58 @@ components: description: Whether the DAG is SubDAG. type: boolean readOnly: true + last_parsed_time: + type: string + format: date-time + readOnly: true + nullable: true + description: | + The last time the DAG was parsed. + + *New in version 2.3.0* + last_pickled: + type: string + format: date-time + readOnly: true + nullable: true + description: | + The last time the DAG was pickled. + + *New in version 2.3.0* + last_expired: + type: string + format: date-time + readOnly: true + nullable: true + description: | + Time when the DAG last received a refresh signal + (e.g. the DAG's "refresh" button was clicked in the web UI) + + *New in version 2.3.0* + scheduler_lock: + type: boolean + readOnly: true + nullable: true + description: | + Whether (one of) the scheduler is scheduling this DAG at the moment + + *New in version 2.3.0* + pickle_id: + type: string + readOnly: true + nullable: true + description: | + Foreign key to the latest pickle_id + + *New in version 2.3.0* + default_view: + type: string + nullable: true + readOnly: true + description: | + Default view of the DAG inside the webserver + + *New in version 2.3.0* fileloc: description: The absolute path to the file. type: string @@ -2136,6 +2188,14 @@ components: describe DAG contents. schedule_interval: $ref: '#/components/schemas/ScheduleInterval' + timetable_description: + type: string + readOnly: true + nullable: true + description: | + Timetable/Schedule Interval description. + + *New in version 2.3.0* tags: description: List of tags. type: array @@ -2143,6 +2203,74 @@ components: items: $ref: '#/components/schemas/Tag' readOnly: true + max_active_tasks: + type: integer + nullable: true + readOnly: true + description: | + Maximum number of active tasks that can be run on the DAG + + *New in version 2.3.0* + max_active_runs: + type: integer + nullable: true + readOnly: true + description: | + Maximum number of active DAG runs for the DAG + + *New in version 2.3.0* + has_task_concurrency_limits: + type: boolean + nullable: true + readOnly: true + description: | + Whether the DAG has task concurrency limits + + *New in version 2.3.0* + has_import_errors: + type: boolean + nullable: true + readOnly: true + description: | + Whether the DAG has import errors + + *New in version 2.3.0* + next_dagrun: + type: string + format: date-time + readOnly: true + nullable: true + description: | + The logical date of the next dag run. + + *New in version 2.3.0* + next_dagrun_data_interval_start: + type: string + format: date-time + readOnly: true + nullable: true + description: | + The start of the interval of the next dag run. + + *New in version 2.3.0* + next_dagrun_data_interval_end: + type: string + format: date-time + readOnly: true + nullable: true + description: | + The end of the interval of the next dag run. + + *New in version 2.3.0* + next_dagrun_create_after: + type: string + format: date-time + readOnly: true + nullable: true + description: | + Earliest time at which this ``next_dagrun`` can be created. + + *New in version 2.3.0* DAGCollection: description: | diff --git a/airflow/api_connexion/schemas/dag_schema.py b/airflow/api_connexion/schemas/dag_schema.py index 3a543bbb86410..373e0ff54f66d 100644 --- a/airflow/api_connexion/schemas/dag_schema.py +++ b/airflow/api_connexion/schemas/dag_schema.py @@ -51,12 +51,27 @@ class Meta: is_paused = auto_field() is_active = auto_field(dump_only=True) is_subdag = auto_field(dump_only=True) + last_parsed_time = auto_field(dump_only=True) + last_pickled = auto_field(dump_only=True) + last_expired = auto_field(dump_only=True) + scheduler_lock = auto_field(dump_only=True) + pickle_id = auto_field(dump_only=True) + default_view = auto_field(dump_only=True) fileloc = auto_field(dump_only=True) file_token = fields.Method("get_token", dump_only=True) owners = fields.Method("get_owners", dump_only=True) description = auto_field(dump_only=True) schedule_interval = fields.Nested(ScheduleIntervalSchema) + timetable_description = auto_field(dump_only=True) tags = fields.List(fields.Nested(DagTagSchema), dump_only=True) + max_active_tasks = auto_field(dump_only=True) + max_active_runs = auto_field(dump_only=True) + has_task_concurrency_limits = auto_field(dump_only=True) + has_import_errors = auto_field(dump_only=True) + next_dagrun = auto_field(dump_only=True) + next_dagrun_data_interval_start = auto_field(dump_only=True) + next_dagrun_data_interval_end = auto_field(dump_only=True) + next_dagrun_create_after = auto_field(dump_only=True) @staticmethod def get_owners(obj: DagModel): diff --git a/airflow/models/dag.py b/airflow/models/dag.py index 5ccc70438d9a0..0eacb8d15efd4 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -2689,7 +2689,7 @@ class DagModel(Base): owners = Column(String(2000)) # Description of the dag description = Column(Text) - # Default view of the inside the webserver + # Default view of the DAG inside the webserver default_view = Column(String(25)) # Schedule interval schedule_interval = Column(Interval) diff --git a/tests/api_connexion/endpoints/test_dag_endpoint.py b/tests/api_connexion/endpoints/test_dag_endpoint.py index 82a2403f2fb4d..41bbb4ca7bba8 100644 --- a/tests/api_connexion/endpoints/test_dag_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_endpoint.py @@ -155,6 +155,21 @@ def test_should_respond_200(self): "root_dag_id": None, "schedule_interval": {"__type": "CronExpression", "value": "2 2 * * *"}, "tags": [], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': False, + 'pickle_id': None, } == response.json @conf_vars({("webserver", "secret_key"): "mysecret"}) @@ -181,6 +196,21 @@ def test_should_respond_200_with_schedule_interval_none(self, session): "root_dag_id": None, "schedule_interval": None, "tags": [], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': False, + 'pickle_id': None, } == response.json def test_should_respond_200_with_granular_dag_access(self): @@ -254,6 +284,8 @@ def test_should_respond_200(self): "start_date": "2020-06-15T00:00:00+00:00", "tags": [{'name': 'example'}], "timezone": "Timezone('UTC')", + "max_active_runs": 16, + "pickle_id": None, } assert response.json == expected @@ -288,6 +320,8 @@ def test_should_response_200_with_doc_md_none(self): "start_date": "2020-06-15T00:00:00+00:00", "tags": [], "timezone": "Timezone('UTC')", + "max_active_runs": 16, + "pickle_id": None, } assert response.json == expected @@ -322,6 +356,8 @@ def test_should_response_200_for_null_start_date(self): "start_date": None, "tags": [], "timezone": "Timezone('UTC')", + "max_active_runs": 16, + "pickle_id": None, } assert response.json == expected @@ -367,6 +403,8 @@ def test_should_respond_200_serialized(self): "start_date": "2020-06-15T00:00:00+00:00", "tags": [{'name': 'example'}], "timezone": "Timezone('UTC')", + "max_active_runs": 16, + "pickle_id": None, } response = self.client.get( f"/api/v1/dags/{self.dag_id}/details", environ_overrides={'REMOTE_USER': "test"} @@ -409,6 +447,8 @@ def test_should_respond_200_serialized(self): 'start_date': '2020-06-15T00:00:00+00:00', 'tags': [{'name': 'example'}], 'timezone': "Timezone('UTC')", + "max_active_runs": 16, + "pickle_id": None, } assert response.json == expected @@ -461,6 +501,21 @@ def test_should_respond_200(self, session): "value": "2 2 * * *", }, "tags": [], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': False, + 'pickle_id': None, }, { "dag_id": "TEST_DAG_2", @@ -477,6 +532,21 @@ def test_should_respond_200(self, session): "value": "2 2 * * *", }, "tags": [], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': False, + 'pickle_id': None, }, ], "total_entries": 2, @@ -505,6 +575,21 @@ def test_only_active_true_returns_active_dags(self): "value": "2 2 * * *", }, "tags": [], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': False, + 'pickle_id': None, } ], "total_entries": 1, @@ -534,6 +619,21 @@ def test_only_active_false_returns_all_dags(self): "value": "2 2 * * *", }, "tags": [], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': False, + 'pickle_id': None, }, { "dag_id": "TEST_DAG_DELETED_1", @@ -550,6 +650,21 @@ def test_only_active_false_returns_all_dags(self): "value": "2 2 * * *", }, "tags": [], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': False, + 'pickle_id': None, }, ], "total_entries": 2, @@ -709,6 +824,21 @@ def test_should_respond_200_on_patch_is_paused(self): "value": "2 2 * * *", }, "tags": [], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': False, + 'pickle_id': None, } assert response.json == expected_response @@ -791,6 +921,21 @@ def test_should_respond_200_with_update_mask(self): "value": "2 2 * * *", }, "tags": [], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': False, + 'pickle_id': None, } assert response.json == expected_response @@ -875,6 +1020,21 @@ def test_should_respond_200_on_patch_is_paused(self, session): "value": "2 2 * * *", }, "tags": [], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': False, + 'pickle_id': None, }, { "dag_id": "TEST_DAG_2", @@ -891,6 +1051,21 @@ def test_should_respond_200_on_patch_is_paused(self, session): "value": "2 2 * * *", }, "tags": [], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': False, + 'pickle_id': None, }, ], "total_entries": 2, @@ -924,6 +1099,21 @@ def test_only_active_true_returns_active_dags(self): "value": "2 2 * * *", }, "tags": [], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': False, + 'pickle_id': None, } ], "total_entries": 1, @@ -959,6 +1149,36 @@ def test_only_active_false_returns_all_dags(self): "value": "2 2 * * *", }, "tags": [], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': False, + 'pickle_id': None, + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': False, + 'pickle_id': None, }, { "dag_id": "TEST_DAG_DELETED_1", @@ -975,6 +1195,21 @@ def test_only_active_false_returns_all_dags(self): "value": "2 2 * * *", }, "tags": [], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': False, + 'pickle_id': None, }, ], "total_entries": 2, @@ -1169,6 +1404,21 @@ def test_should_respond_200_and_pause_dags(self): "value": "2 2 * * *", }, "tags": [], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': False, + 'pickle_id': None, }, { "dag_id": "TEST_DAG_2", @@ -1185,6 +1435,21 @@ def test_should_respond_200_and_pause_dags(self): "value": "2 2 * * *", }, "tags": [], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': False, + 'pickle_id': None, }, ], "total_entries": 2, @@ -1221,6 +1486,21 @@ def test_should_respond_200_and_pause_dag_pattern(self, session): "value": "2 2 * * *", }, "tags": [], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': False, + 'pickle_id': None, }, { "dag_id": "TEST_DAG_10", @@ -1237,6 +1517,21 @@ def test_should_respond_200_and_pause_dag_pattern(self, session): "value": "2 2 * * *", }, "tags": [], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': False, + 'pickle_id': None, }, ], "total_entries": 2, diff --git a/tests/api_connexion/schemas/test_dag_schema.py b/tests/api_connexion/schemas/test_dag_schema.py index f12c06d036593..6161b21476753 100644 --- a/tests/api_connexion/schemas/test_dag_schema.py +++ b/tests/api_connexion/schemas/test_dag_schema.py @@ -48,6 +48,7 @@ def test_serialize(self): tags=[DagTag(name="tag-1"), DagTag(name="tag-2")], ) serialized_dag = DAGSchema().dump(dag_model) + assert { "dag_id": "test_dag_id", "description": "The description", @@ -60,6 +61,21 @@ def test_serialize(self): "root_dag_id": "test_root_dag_id", "schedule_interval": {"__type": "CronExpression", "value": "5 4 * * *"}, "tags": [{"name": "tag-1"}, {"name": "tag-2"}], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': None, + 'pickle_id': None, } == serialized_dag @@ -83,6 +99,21 @@ def test_serialize(self): "root_dag_id": None, "schedule_interval": None, "tags": [], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': None, + 'pickle_id': None, }, { "dag_id": "test_dag_id_b", @@ -96,6 +127,21 @@ def test_serialize(self): "root_dag_id": None, "schedule_interval": None, "tags": [], + 'next_dagrun': None, + 'has_task_concurrency_limits': True, + 'next_dagrun_data_interval_start': None, + 'next_dagrun_data_interval_end': None, + 'max_active_runs': 16, + 'next_dagrun_create_after': None, + 'last_expired': None, + 'max_active_tasks': 16, + 'last_pickled': None, + 'default_view': None, + 'last_parsed_time': None, + 'scheduler_lock': None, + 'timetable_description': None, + 'has_import_errors': None, + 'pickle_id': None, }, ], "total_entries": 2, @@ -142,5 +188,7 @@ def test_serialize(self): 'start_date': '2020-06-19T00:00:00+00:00', 'tags': [{'name': "example1"}, {'name': "example2"}], 'timezone': "Timezone('UTC')", + 'max_active_runs': 16, + 'pickle_id': None, } assert schema.dump(dag) == expected