[EWT-569] Airflow Upgrade to 1.10.14 [CP] from 1.10.4+twtr : 1991419

[CP][TWTR][EWT-377] Fix DagBag bug when a Dag has invalid schedule_interval (twitter-forks#61)

CP of 5605d10
& apache#11462

airflow/jobs/scheduler_job.py

@@ -1442,6 +1442,7 @@ def _execute_helper(self):
         while (timezone.utcnow() - execute_start_time).total_seconds() < \
                 self.run_duration or self.run_duration < 0:
             self.log.debug("Starting Loop...")
+            execute_start_time = timezone.utcnow()
             loop_start_time = time.time()
 
             if self.using_sqlite:
@@ -1525,6 +1526,16 @@ def _execute_helper(self):
                 self.log.debug("Sleeping for %.2f seconds", self._processor_poll_interval)
                 time.sleep(self._processor_poll_interval)
 
+            # Verify that all files were processed, and if so, deactivate DAGs that
+            # haven't been touched by the scheduler as they likely have been
+            # deleted.
+            if self.processor_agent.all_files_processed:
+                self.log.info(
+                    "Deactivating DAGs that haven't been touched since %s",
+                    execute_start_time.isoformat()
+                )
+                models.DAG.deactivate_stale_dags(execute_start_time)
+
             if self.processor_agent.done:
                 self.log.info("Exiting scheduler loop as all files"
                               " have been processed {} times".format(self.num_runs))
@@ -1542,16 +1553,6 @@ def _execute_helper(self):
         self.processor_agent.terminate()
         self.log.info("All DAG processors terminated")
 
-        # Verify that all files were processed, and if so, deactivate DAGs that
-        # haven't been touched by the scheduler as they likely have been
-        # deleted.
-        if self.processor_agent.all_files_processed:
-            self.log.info(
-                "Deactivating DAGs that haven't been touched since %s",
-                execute_start_time.isoformat()
-            )
-            models.DAG.deactivate_stale_dags(execute_start_time)
-
         self.executor.end()
 
         settings.Session.remove()

tests/models/test_dagbag.py

@@ -171,6 +171,7 @@ def test_process_file_cron_validity_check(self):
         for d in invalid_dag_files:
             dagbag.process_file(os.path.join(TEST_DAGS_FOLDER, d))
         self.assertEqual(len(dagbag.import_errors), len(invalid_dag_files))
+        self.assertEqual(len(dagbag.dags), 0)
 
     @patch.object(DagModel, 'get_current')
     def test_get_dag_without_refresh(self, mock_dagmodel):