From eda6fd3bb336bafbae99a3a6871ff407d1fca409 Mon Sep 17 00:00:00 2001 From: ravitezu Date: Wed, 14 Oct 2020 22:45:07 +0530 Subject: [PATCH 1/3] add upgrade rule to check for mesos config and flag to remove it. --- .../upgrade/rules/mesos_executor_removed.py | 35 +++++++++++++++++++ .../rules/test_mesos_executor_removed.py | 34 ++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 airflow/upgrade/rules/mesos_executor_removed.py create mode 100644 tests/upgrade/rules/test_mesos_executor_removed.py diff --git a/airflow/upgrade/rules/mesos_executor_removed.py b/airflow/upgrade/rules/mesos_executor_removed.py new file mode 100644 index 000000000000..d8414cb95c51 --- /dev/null +++ b/airflow/upgrade/rules/mesos_executor_removed.py @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from airflow.upgrade.rules.base_rule import BaseRule +from airflow.configuration import conf + + +class MesosExecutorRemovedRule(BaseRule): + """ + MesosExecutorRemovedRule class to ease upgrade to Airflow 2.0 + """ + + title = "Removal of Mesos Executor" + description = """ + The Mesos Executor has been deprecated as it was not widely used and not maintained. + """ + + def check(self): + mesos_config = conf.getsection("mesos") + if mesos_config: + return "The Mesos Executor has been deprecated as it was not widely used and not maintained." diff --git a/tests/upgrade/rules/test_mesos_executor_removed.py b/tests/upgrade/rules/test_mesos_executor_removed.py new file mode 100644 index 000000000000..7df257d423e1 --- /dev/null +++ b/tests/upgrade/rules/test_mesos_executor_removed.py @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from unittest import TestCase + +from airflow.upgrade.rules.mesos_executor_removed import MesosExecutorRemovedRule +from tests.test_utils.config import conf_vars + + +class TestMesosExecutorRemovedRule(TestCase): + @conf_vars({("mesos", "description"): "mesos description"}) + def test_check(self): + rule = MesosExecutorRemovedRule() + + assert isinstance(rule.description, str) + assert isinstance(rule.title, str) + + msg = "The Mesos Executor has been deprecated as it was not widely used and not maintained." + + response = rule.check() + assert response == msg From 3bd0de875c88d8c24641071fb111c1f228c9570a Mon Sep 17 00:00:00 2001 From: ravitezu Date: Thu, 15 Oct 2020 11:57:38 +0530 Subject: [PATCH 2/3] change from checking the mesos config section to core/executor config --- .../upgrade/rules/mesos_executor_removed.py | 10 +++++++--- .../rules/test_mesos_executor_removed.py | 20 ++++++++++++++++--- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/airflow/upgrade/rules/mesos_executor_removed.py b/airflow/upgrade/rules/mesos_executor_removed.py index d8414cb95c51..6daebc51e84f 100644 --- a/airflow/upgrade/rules/mesos_executor_removed.py +++ b/airflow/upgrade/rules/mesos_executor_removed.py @@ -30,6 +30,10 @@ class MesosExecutorRemovedRule(BaseRule): """ def check(self): - mesos_config = conf.getsection("mesos") - if mesos_config: - return "The Mesos Executor has been deprecated as it was not widely used and not maintained." + executor_key = conf.get(section="core", key="executor") + if executor_key == "MesosExecutor": + return ( + "The Mesos Executor has been deprecated as it was not widely used and not maintained." + "Please migrate to any of the supported executors." + "See https://airflow.apache.org/docs/stable/executor/index.html for more details." + ) diff --git a/tests/upgrade/rules/test_mesos_executor_removed.py b/tests/upgrade/rules/test_mesos_executor_removed.py index 7df257d423e1..2b1e53065153 100644 --- a/tests/upgrade/rules/test_mesos_executor_removed.py +++ b/tests/upgrade/rules/test_mesos_executor_removed.py @@ -21,14 +21,28 @@ class TestMesosExecutorRemovedRule(TestCase): - @conf_vars({("mesos", "description"): "mesos description"}) - def test_check(self): + @conf_vars({("core", "executor"): "MesosExecutor"}) + def test_invalid_check(self): rule = MesosExecutorRemovedRule() assert isinstance(rule.description, str) assert isinstance(rule.title, str) - msg = "The Mesos Executor has been deprecated as it was not widely used and not maintained." + msg = ( + "The Mesos Executor has been deprecated as it was not widely used and not maintained." + "Please migrate to any of the supported executors." + "See https://airflow.apache.org/docs/stable/executor/index.html for more details." + ) response = rule.check() assert response == msg + + @conf_vars({("core", "executor"): "SequentialExecutor"}) + def test_check(self): + rule = MesosExecutorRemovedRule() + + assert isinstance(rule.description, str) + assert isinstance(rule.title, str) + + response = rule.check() + assert response is None From dbca38fb53773dd39d8d34d1190b0d986e1090a9 Mon Sep 17 00:00:00 2001 From: ravitezu Date: Tue, 20 Oct 2020 01:04:53 +0530 Subject: [PATCH 3/3] remove leading new line and indent in desc --- airflow/upgrade/rules/mesos_executor_removed.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/airflow/upgrade/rules/mesos_executor_removed.py b/airflow/upgrade/rules/mesos_executor_removed.py index 6daebc51e84f..c0e6b52efdeb 100644 --- a/airflow/upgrade/rules/mesos_executor_removed.py +++ b/airflow/upgrade/rules/mesos_executor_removed.py @@ -23,11 +23,8 @@ class MesosExecutorRemovedRule(BaseRule): """ MesosExecutorRemovedRule class to ease upgrade to Airflow 2.0 """ - title = "Removal of Mesos Executor" - description = """ - The Mesos Executor has been deprecated as it was not widely used and not maintained. - """ + description = "The Mesos Executor has been deprecated as it was not widely used and not maintained." def check(self): executor_key = conf.get(section="core", key="executor")