From cc8519d1bedd665d89834343171ecf62e72d8f2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= <6774676+eumiro@users.noreply.github.com> Date: Thu, 10 Aug 2023 22:54:13 +0000 Subject: [PATCH] Refactor: Simplify code in utils (#33268) --- airflow/utils/file.py | 3 +-- airflow/utils/log/file_task_handler.py | 11 +++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/airflow/utils/file.py b/airflow/utils/file.py index fb17ee6a8d57..2b230b61290b 100644 --- a/airflow/utils/file.py +++ b/airflow/utils/file.py @@ -22,7 +22,6 @@ import logging import os import zipfile -from collections import OrderedDict from pathlib import Path from typing import Generator, NamedTuple, Pattern, Protocol, overload @@ -230,7 +229,7 @@ def _find_path_from_directory( ] # evaluation order of patterns is important with negation # so that later patterns can override earlier patterns - patterns = list(OrderedDict.fromkeys(patterns).keys()) + patterns = list(dict.fromkeys(patterns)) dirs[:] = [subdir for subdir in dirs if not ignore_rule_type.match(Path(root) / subdir, patterns)] diff --git a/airflow/utils/log/file_task_handler.py b/airflow/utils/log/file_task_handler.py index 6c8073b005b6..2a7bd688b5ae 100644 --- a/airflow/utils/log/file_task_handler.py +++ b/airflow/utils/log/file_task_handler.py @@ -491,12 +491,11 @@ def _init_file(self, ti): @staticmethod def _read_from_local(worker_log_path: Path) -> tuple[list[str], list[str]]: messages = [] - logs = [] - files = list(worker_log_path.parent.glob(worker_log_path.name + "*")) - if files: - messages.extend(["Found local files:", *[f" * {x}" for x in sorted(files)]]) - for file in sorted(files): - logs.append(Path(file).read_text()) + paths = sorted(worker_log_path.parent.glob(worker_log_path.name + "*")) + if paths: + messages.append("Found local files:") + messages.extend(f" * {x}" for x in paths) + logs = [file.read_text() for file in paths] return messages, logs def _read_from_logs_server(self, ti, worker_log_rel_path) -> tuple[list[str], list[str]]: