Add automatic backup creation for download requirements (#2991) (#2994)

* Add automatic backup creation for download requirements (#2991)

* add backup creation mechenism

* fix prepare dir name

ansible/playbooks/roles/repository/files/download-requirements/src/command/tar.py

@@ -13,66 +13,91 @@ def __init__(self):
         super().__init__('tar', 1)
 
     def pack(self, filename: Path,
-             target: str,
-             directory: Path = None,
-             verbose: bool = False,
+             targets: List[Path],
+
+             # short flags:
              compress: bool = False,
+             verbose: bool = False,
+             preserve: bool = False,
+
+             # long flags:
+             absolute_names: bool = False,
+             directory: Path = None,
              verify: bool = False):
         """
-        Create a tar archive
+        Create a tar archive.
 
         :param filename: name for the archive to be created
-        :param target: files to be archived
-        :param directory: change directory before doing any actions
+        :param targets: files to be archived
+
+        :param compress: use zlib compression
         :param verbose: use verbose mode
-        :param uncompress: use zlib compression
+        :param preserve: extract information about file permissions
+
+        :param absolute_names: don't strip leading slashes from file names
+        :param directory: change directory before doing any actions
         :param verify: check file integrity
         """
         short_flags: List[str] = ['-c']  # -czvf flags
         tar_params: List[str] = [str(filename)]  # all the other params
 
+        # short flags:
         if compress:
             short_flags.append('z')
 
         if verbose:
             short_flags.append('v')
 
+        if preserve:
+            short_flags.append('p')
+
         short_flags.append('f')
 
-        if verify:
-            tar_params.append('--verify')
+        # long flags:
+        if absolute_names:
+            tar_params.append('--absolute-names')
 
         if directory is not None:
             tar_params.extend(['--directory', str(directory)])
 
-        if target:
-            tar_params.append(target)
+        if verify:
+            tar_params.append('--verify')
+
+        for target in targets:
+            tar_params.append(str(target))
 
         self.run([''.join(short_flags)] + tar_params)
 
     def unpack(self, filename: Path,
-               target: str = '',
+               target: Path = None,
+
+               # short flags:
+               uncompress: bool = True,
+               verbose: bool = False,
+
+               # long flags:
                absolute_names: bool = False,
                directory: Path = None,
                overwrite: bool = True,
-               strip_components: int = 0,
-               uncompress: bool = True,
-               verbose: bool = False):
+               strip_components: int = 0):
         """
-        Unpack a tar archive
+        Unpack a tar archive.
 
         :param filename: file to be extracted
         :param target: name for the output file
+
+        :param uncompress: use zlib compression
+        :param verbose: use verbose mode
+
         :param absolute_names: use abs path names
         :param directory: change directory before doing any actions
         :param overwrite: overwrite existing files when extracting
         :param strip_components: strip leading components from file names on extraction
-        :param uncompress: use zlib compression
-        :param verbose: use verbose mode
         """
         short_flags: List[str] = ['-x']  # -xzvf flags
         tar_params: List[str] = [str(filename)]  # all the other params
 
+        # short flags
         if uncompress:
             short_flags.append('z')
 
@@ -81,6 +106,7 @@ def unpack(self, filename: Path,
 
         short_flags.append('f')
 
+        # long flags
         if absolute_names:
             tar_params.append('--absolute-names')
 
@@ -90,10 +116,10 @@ def unpack(self, filename: Path,
         if strip_components:
             tar_params.append(f'--strip-components={str(strip_components)}')
 
-        if target:
-            tar_params.append(target)
-
         if overwrite:
             tar_params.append('--overwrite')
 
+        if target is not None:
+            tar_params.append(str(target))
+
         self.run([''.join(short_flags)] + tar_params)

ansible/playbooks/roles/repository/files/download-requirements/src/config.py

@@ -34,7 +34,6 @@ def __init__(self, argv: List[str]):
         self.dest_images: Path
         self.dest_packages: Path
         self.distro_subdir: Path
-        self.enable_backup: bool
         self.is_log_file_enabled: bool
         self.log_file: Path
         self.os_arch: OSArch
@@ -47,6 +46,7 @@ def __init__(self, argv: List[str]):
         self.rerun: bool
         self.retries: int
         self.script_path: Path
+        self.was_backup_created: bool = False
 
         self.__add_args(argv)
 
@@ -70,10 +70,7 @@ def __log_info_summary(self):
         lines.append(f'- grafana dashboards: {str(self.dest_grafana_dashboards)}')
         lines.append(f'- images:             {str(self.dest_images)}')
         lines.append(f'- packages:           {str(self.dest_packages)}')
-
-        lines.append(f'Enable repos backup: {"Yes" if self.enable_backup else "No"}')
-        if self.enable_backup:
-            lines.append(f'Repos backup file: {str(self.repos_backup_file)}')
+        lines.append(f'Repos backup file: {str(self.repos_backup_file)}')
 
         if self.is_log_file_enabled:
             lines.append(f'Log file location: {str(self.log_file.absolute())}')
@@ -97,8 +94,6 @@ def __create_parser(self) -> ArgumentParser:
                             'when using `detect`, script will try to find out which OS is being used')
 
         # optional arguments:
-        parser.add_argument('--enable-repos-backup', '-b', action='store_true', dest='enable_backup', default=False,
-                            help=('when used, backup archive for packages will be created and used')),
         parser.add_argument('--repos-backup-file', metavar='BACKUP_FILE', action='store',
                             dest='repos_backup_file', default='/var/tmp/enabled-system-repos.tar',
                             help='path to a backup file')
@@ -109,7 +104,7 @@ def __create_parser(self) -> ArgumentParser:
                             default=Path('./download-requirements.log'),
                             help='logs will be saved to this file')
         parser.add_argument('--log-level', metavar='LOG_LEVEL', type=str, action='store', dest='log_level',
-                            default='info', help='set up log level, available levels: (error|warn|info|debug`)')
+                            default='info', help='set up log level, available levels: (error|warn|info|debug)')
         parser.add_argument('--no-logfile', action='store_true', dest='no_logfile',
                             help='no logfile will be created')
 
@@ -207,7 +202,6 @@ def __add_args(self, argv: List[str]):
         self.dest_packages = self.dest_dir / 'packages'
 
         # add optional arguments
-        self.enable_backup = args['enable_backup']
         self.os_arch = OSArch(os.uname().machine)
         self.repos_backup_file = Path(args['repos_backup_file'])
         self.retries = args['retries']

ansible/playbooks/roles/repository/files/download-requirements/src/mode/base_mode.py

@@ -71,15 +71,15 @@ def __parse_requirements(self) -> Dict[str, Any]:
 
         return reqs
 
-    def _use_backup_repositories(self):
+    def _create_backup_repositories(self):
         """
-        Check if there were any critical issues and if so, try to restore the state using backup
+        Create a backup of package repository files under the /etc directory.
         """
         raise NotImplementedError
 
     def _add_third_party_repositories(self):
         """
-        Add third party repositories for target OS's package manager
+        Add third party repositories for target OS's package manager.
         """
         raise NotImplementedError
 
@@ -91,7 +91,7 @@ def _install_base_packages(self):
 
     def _download_packages(self):
         """
-        Download packages under `self._requirements['packages']` using target OS's package manager
+        Download packages under `self._requirements['packages']` using target OS's package manager.
         """
         raise NotImplementedError
 
@@ -140,7 +140,7 @@ def __download_files(self):
 
     def __download_grafana_dashboards(self):
         """
-        Download grafana dashboards under `self._requirements['grafana-dashboards']`
+        Download grafana dashboards under `self._requirements['grafana-dashboards']`.
         """
         dashboards: Dict[str, Dict] = self._requirements['grafana-dashboards']
         for dashboard in dashboards:
@@ -158,7 +158,7 @@ def __download_grafana_dashboards(self):
 
     def __download_crane(self):
         """
-        Download Crane package if needed and setup it's environment
+        Download Crane package if needed and setup it's environment.
         """
         crane_path = self._cfg.dest_dir / 'crane'
         crane_package_path = Path(f'{crane_path}.tar.gz')
@@ -169,7 +169,7 @@ def __download_crane(self):
             logging.debug('crane - checksum ok, skipped')
         else:
             self._download_crane_binary(first_crane, crane_package_path)
-            self._tools.tar.unpack(crane_package_path, 'crane', directory=self._cfg.dest_dir)
+            self._tools.tar.unpack(crane_package_path, Path('crane'), directory=self._cfg.dest_dir)
             chmod(crane_path, 0o0755)
 
         # create symlink to the crane file so that it'll be visible in shell
@@ -180,7 +180,7 @@ def __download_crane(self):
 
     def _download_images(self):
         """
-        Download images under `self._requirements['images']` using Crane
+        Download images under `self._requirements['images']` using Crane.
         """
         platform: str = 'linux/amd64' if self._cfg.os_arch.X86_64 else 'linux/arm64'
         images = self._requirements['images']
@@ -200,10 +200,23 @@ def _download_images(self):
 
     def _cleanup(self):
         """
-        Optional step for cleanup routines
+        Optional step for cleanup routines.
         """
         pass
 
+    def __restore_repositories(self):
+        """
+        Restore the state of repository files under the /etc dir.
+        """
+        if self._cfg.repos_backup_file.exists() and self._cfg.repos_backup_file.stat().st_size:
+            logging.info('Restoring repository files...')
+            self._tools.tar.unpack(filename=self._cfg.repos_backup_file,
+                                   directory=Path('/'),
+                                   absolute_names=True,
+                                   uncompress=False,
+                                   verbose=True)
+            logging.info('Done restoring repository files.')
+
     def run(self):
         """
         Run target mode.
@@ -218,9 +231,10 @@ def run(self):
         self._cfg.dest_images.mkdir(exist_ok=True, parents=True)
         self._cfg.dest_packages.mkdir(exist_ok=True, parents=True)
 
-        logging.info('Checking backup repositories...')
-        self._use_backup_repositories()
-        logging.info('Done checking backup repositories.')
+        self._create_backup_repositories()
+
+        if not self._cfg.was_backup_created:
+            self.__restore_repositories()
 
         logging.info('Installing base packages...')
         self._install_base_packages()
@@ -259,3 +273,5 @@ def run(self):
         logging.info('Running cleanup...')
         self._cleanup()
         logging.info('Done running cleanup.')
+
+        self.__restore_repositories()

ansible/playbooks/roles/repository/files/download-requirements/src/mode/debian_family_mode.py

@@ -22,19 +22,19 @@ def __create_repo_paths(self):
         for repo in self._repositories.keys():
             self._repositories[repo]['path'] = Path('/etc/apt/sources.list.d') / f'{repo}.list'
 
-    def _use_backup_repositories(self):
-        sources = Path('/etc/apt/sources.list')
-        if not sources.exists() or not sources.stat().st_size:
-            if self._cfg.repos_backup_file.exists() and self._cfg.enable_backup:
-                logging.warn('OS repositories seems missing, restoring...')
-                self._tools.tar.unpack(filename=self._cfg.repos_backup_file,
-                                       directory=Path('/'),
-                                       absolute_names=True,
-                                       uncompress=False,
-                                       verbose=True)
-            else:
-                logging.warn(f'{str(sources)} seems to be missing, you either know what you are doing or '
-                             'you need to fix your repositories')
+    def _create_backup_repositories(self):
+        if not self._cfg.repos_backup_file.exists():
+            logging.debug('Creating backup for system repositories...')
+            self._tools.tar.pack(self._cfg.repos_backup_file,
+                                 targets=[Path('/etc/apt/sources.list'),
+                                          Path('/etc/apt/sources.list.d')],
+                                 verbose=True,
+                                 preserve=True,
+                                 absolute_names=True,
+                                 verify=True)
+
+            self._cfg.was_backup_created = True
+            logging.debug('Done.')
 
     def _install_base_packages(self):
         # install prerequisites which might be missing
@@ -47,11 +47,6 @@ def _install_base_packages(self):
                 logging.info(f'- {package}')
 
     def _add_third_party_repositories(self):
-        # backup custom repositories to avoid possible conflicts
-        for repo_file in Path('/etc/apt/sources.list.d').iterdir():
-            if repo_file.name.endswith('.list'):
-                repo_file.rename(f'{repo_file}.bak')
-
         # add third party keys
         for repo in self._repositories:
             data = self._repositories[repo]
@@ -125,10 +120,9 @@ def _cleanup(self):
             if data['path'].exists():
                 data['path'].unlink()
 
-        # restore masked custom repositories to their original names
+        # removed custom repositories to their original names
         for repo_file in Path('/etc/apt/sources.list.d').iterdir():
-            if repo_file.name.endswith('.bak'):
-                move(str(repo_file.absolute()), str(repo_file.with_suffix('').absolute()))
+            repo_file.unlink()
 
         # remove installed packages
         for package in self.__installed_packages: