Merge branch 'release-0.18.0'

configsnap

@@ -24,11 +24,8 @@ import glob
 import re
 import tarfile
 
-
 version = "0.17.1"
-diffs_found_msg = False
-is_php_detected = False
-is_apache_detected = False
+diffs_found = False
 
 
 def report_verbose(message):
@@ -128,98 +125,73 @@ def create_archive(tagdir, workdir, tag, overwrite):
     return output_filename, True
 
 
-def compare_files():
-    # Check that there are pre files, and exit if there aren't
-    found_pre = False
+def compare_files(options):
+    """Compare 2 phases of a same tag"""
     run = dataGather(workdir, phase)
 
-    for filename in os.listdir(workdir):
-        if filename.endswith(options.pre_suffix):
-            found_pre = True
-    if not found_pre:
-        report_error("Unable to diff, as no " +
-                     options.pre_suffix + " pre files")
-        sys.exit(0)
-
-    report_info("\nThis is %s phase so performing file diffs..." % phase)
-
-    if 'rollback' in phase:
-        run.get_diff_files('packages')
-
-    compare_files = ['sysvinit', 'mounts', 'netstat', 'ip_addresses',
-                     'network', 'interfaces', 'hosts', 'lvs', 'pvs', 'vgs',
-                     'fstab', 'powermt', 'omreport_version', 'ip_routes',
-                     'partitions', 'lspci', 'omreport_memory',
-                     'omreport_processors', 'omreport_nics', 'omreport_vdisk',
-                     'omreport_pdisk', 'hpreport_server', 'hpreport_memory',
-                     'hpreport_bios', 'hpreport_storage_controller',
-                     'hpreport_storage_vdisk', 'hpreport_storage_pdisk',
-                     'multipath', 'systemdinit', 'upstartinit', 'yum.conf',
-                     'dnf.conf', 'pcs_constraints', 'sudoers']
-
-    # Add php files if detected
-    if is_php_detected:
-        compare_files.append('php.ini')
-        if os.path.exists('/usr/bin/pecl'):
-            compare_files.append('pecl-list')
-
-    # Add apache files if detected
-    if is_apache_detected:
-        compare_files.append('httpd_modules')
-        compare_files.append('httpd_vhosts')
-
-    # Array to hold custom collection files and commands to compare
-    additional_to_compare = []
-    for section in Config.sections():
-        try:
-            if Config.has_option(section, 'compare') and Config.get(section, 'compare').lower() == "true":
-                additional_to_compare.append(section)
-        except ConfigParser.NoOptionError, e:
-            report_error(
-                "Check config file options for section %s: %s" % (section, e))
-        except Exception, e:
-            report_error(
-                "Could not parse config file section %s: %s" % (section, e))
+    # Collects all files and dir at the top level
+    all_files = os.listdir(workdir)
+    # Those will always result in positive diffs, this is misleading so we exclude them
+    exclusions = '^(sysctl|vgcfgbackup[-].*|ps|meminfo|dmesg|crm_mon|pcs_status)$'
 
-    compare_files.extend(additional_to_compare)
+    # Proceed if at least one file with options.pre_suffix is present
+    if any(filename.endswith(".%s" % options.pre_suffix) for filename in all_files):
+        report_info("Perfoming file diffs...\n")
 
-    for filename in compare_files:
-        pre_filename = os.path.join(workdir,
-                                    "%s.%s" % (filename, options.pre_suffix))
-        if os.path.exists(pre_filename):
-            run.get_diff_files(filename)
+        if 'rollback' in phase:
+            run.get_diff_files('packages', options)
 
-    # Report subdirectories at the subdir level unless ther verbose option is
-    # given, in which case report as usual
-    report_info("\nStarting sub-dir diffs...")
-    for directory in os.listdir(workdir):
-        if os.path.isdir(os.path.join(workdir, directory)):
-            run.get_diff_dir(directory)
+        # to search for files with pre_suffix or phase only
+        suffix_exist = re.compile('[.](' + options.pre_suffix + '|' + options.phase + ')$')
+        filtered_files = []
+        for index, value in enumerate(all_files):
+            if suffix_exist.search(value):
+                filebase = suffix_exist.sub('', value)
+                if not re.match(exclusions, filebase):
+                    filtered_files.append(filebase)
 
-    try:
-        uname_pre_fd = open(
-            os.path.join(workdir, 'uname.' + options.pre_suffix), 'r')
-        uname_post_fd = open(
-            os.path.join(workdir, "uname.%s" % phase), 'r')
-        uname_pre = uname_pre_fd.read().strip()
-        uname_post = uname_post_fd.read().strip()
-        if uname_pre != uname_post:
-            sys.stdout.write("Old uname: %s. New uname: %s\n" %
-                             (uname_pre, uname_post))
-    except Exception, e:
-        report_error(
-            "Unable to open file %s: %s" % ((e.filename, e.strerror)))
+        compare_files = list(set(filtered_files))
+        compare_files.sort()
 
-    if diffs_found_msg:
-        report_info_blue("\nINTERPRETING DIFFS:\n")
-        report_info_blue("* Lines beginning with a - show an "
-                         "entry from the " + options.pre_suffix + " pre file "
-                         "which has changed\n" "or which is not present in the .post or .rollback file.\n")
-        report_info_blue("* Lines beginning with a + show an entry from the "
-                         ".post or.rollback file which\nhas changed or which "
-                         "is not present in the " + options.pre_suffix + " pre file.\n")
-        report_info_blue("Please review all diff output above carefully and "
-                         "account for any differences\nfound.\n")
+        for filename in compare_files:
+            run.get_diff_files(filename, options)
+
+        # Report subdirectories at the subdir level unless ther verbose option is
+        # given, in which case report as usual
+        report_info("\nStarting sub-dir diffs...")
+        for directory in os.listdir(workdir):
+            if os.path.isdir(os.path.join(workdir, directory)):
+                run.get_diff_dir(directory, options)
+
+        try:
+            uname_pre_fd = open(
+                os.path.join(workdir, 'uname.' + options.pre_suffix), 'r')
+            uname_post_fd = open(
+                os.path.join(workdir, "uname.%s" % phase), 'r')
+            uname_pre = uname_pre_fd.read().strip()
+            uname_post = uname_post_fd.read().strip()
+            if uname_pre != uname_post:
+                sys.stdout.write("Old uname: %s. New uname: %s\n" %
+                                 (uname_pre, uname_post))
+        except Exception, e:
+            report_error(
+                "Unable to open file %s: %s" % ((e.filename, e.strerror)))
+
+        if diffs_found:
+            report_info_blue("\nINTERPRETING DIFFS:\n")
+            report_info_blue("* Lines beginning with a - show an "
+                             "entry from the " + options.pre_suffix + " pre file "
+                             "which has changed\n" "or which is not present in the .post or .rollback file.\n")
+            report_info_blue("* Lines beginning with a + show an entry from the "
+                             ".post or.rollback file which\nhas changed or which "
+                             "is not present in the " + options.pre_suffix + " pre file.\n")
+            report_info_blue("Please review all diff output above carefully and "
+                             "account for any difference found.\n")
+
+    else:
+        report_error("Unable to diff, no files with suffix \"" +
+                     options.pre_suffix + "\" present.")
+        sys.exit(1)
 
 
 class dataGather:
@@ -327,8 +299,7 @@ class dataGather:
             report_error("Unable to open %s: %s" % (filename, e.strerror))
             return False
 
-    def get_diff_files(self, filename):
-        global diffs_found_msg
+    def get_diff_files(self, filename, options):
         filename = os.path.join(self.workdir, filename)
         pre_filename = "%s.%s" % (filename, options.pre_suffix)
         post_filename = "%s.%s" % (filename, self.phase)
@@ -350,16 +321,19 @@ class dataGather:
         diff_stdout, diff_stderr = diff_proc.communicate(input=None)
 
         if diff_proc.returncode != 0:
-            diffs_found_msg = True
+            global diffs_found
+            diffs_found = True
             report_error(
                 "Differences found against %s.%s:\n" % (filename, options.pre_suffix))
-            sys.stdout.writelines(diff_stdout)
+            if diff_stdout != "":
+                sys.stdout.writelines(diff_stdout)
+            else:
+                report_error(diff_stderr)
         else:
             report_info("No differences against %s.%s" %
                         (filename, options.pre_suffix))
 
-    def get_diff_dir(self, sdir):
-        diffs_found_msg = False
+    def get_diff_dir(self, sdir, options):
         all_pre_files = []
         sdir = os.path.join(workdir, sdir)
 
@@ -393,24 +367,46 @@ class dataGather:
                 return None
 
             if diff_proc.wait() != 0:
-                diffs_found_msg = True
+                global diffs_found
+                diffs_found = True
                 report_error("Differences found against %s:\n" % pre_filename)
                 sys.stdout.writelines(diff_proc.stdout.readlines())
             elif options.verbose_enabled is True:
                 report_verbose("No differences against %s" % pre_filename)
 
         # Looking for new files
+        local_diffs = False
         for post_filename in glob.glob(sdir + '/*' + self.phase):
             if post_filename not in all_pre_files:
                 report_error("+ File added: %s:" % post_filename)
-                diffs_found_msg = True
+                local_diffs = True
+
+        if local_diffs is False:
+            report_info("No extra post files found in %s" % sdir)
+
+    def is_running(self, *paths):
+        if len(self.procs_out) == 0:
+            # Get a list of processes running on the server
+            procs = subprocess.Popen(
+                ['ps', '-A', '--noheaders', '-o', 'args'], stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE)
+
+            procs_out, procs_err = procs.communicate()
+            if procs.returncode == 0:
+                self.procs_out = [x.split()[0] for x in procs_out.splitlines()]
+            else:
+                report_error('Failed to get process list')
+
+        for path in paths:
+            if path in self.procs_out:
+                return True
 
-        if diffs_found_msg is False:
-            report_info("No differences found for files in %s" % sdir)
+        return False
 
     def __init__(self, workdir, phase):
         self.workdir = workdir
         self.phase = phase
+        self.procs_out = []
 
 
 ###################################################
@@ -538,7 +534,7 @@ else:
 if options.compare_only:
     report_info("Comparing files from phases: %s and %s" %
                 (options.pre_suffix, options.phase))
-    compare_files()
+    compare_files(options)
     sys.exit(0)
 
 # Ensure each section has a valid Type
@@ -548,19 +544,6 @@ for section in Config.sections():
             report_error("Wrong \"Type\" defined for custom collection entry \"%s\"; "
                          "should be \"file\", \"directory\" or \"command\"" % section)
 
-# Get a list of processes running on the server
-procs = subprocess.Popen(
-    ['ps', '-A', '--noheaders', '-o', 'args'], stdout=subprocess.PIPE,
-    stderr=subprocess.PIPE)
-
-procs_output = procs.communicate()[0]
-if procs.returncode != 0:
-    report_error('Failed to get process list')
-
-# Just keep the process name, drop the arguments
-procs_output = procs_output.splitlines()
-procs_output = [x.split()[0] for x in procs_output]
-
 run = dataGather(workdir, phase)
 report_info("Getting storage details (LVM, partitions, multipathing)...")
 if os.path.exists('/sbin/lvs'):
@@ -717,15 +700,14 @@ if os.path.exists(hpasmpath):
                     break
             break
 
-if '/usr/sbin/httpd' in procs_output or '/usr/sbin/httpd.worker' in procs_output:
+if run.is_running('/usr/sbin/httpd', '/usr/sbin/httpd.worker'):
     report_info("Getting Apache vhosts and modules...")
-    is_apache_detected = True
     run.run_command(
         ['httpd', '-S'], 'httpd_vhosts', fail_ok=False, sort=False)
     run.run_command(
         ['httpd', '-M'], 'httpd_modules', fail_ok=False, sort=True)
 
-if '/usr/libexec/mysqld' in procs_output:
+if run.is_running('/usr/libexec/mysqld', '/usr/sbin/mysqld'):
     report_info("Getting MySQL databases...")
     run.run_command(['mysql', '-Bse', 'show databases;'],
                     'mysql_databases', fail_ok=True, sort=True)
@@ -785,6 +767,24 @@ if os.path.exists('/etc/network/interfaces.d'):
 if os.path.exists('/etc/sysconfig/network-scripts/'):
     run.copy_dir('/etc/sysconfig/network-scripts/', file_pattern='(ifcfg-|route-).*',
                  fail_ok=False, sort=False)
+# php specifics
+if os.path.exists('/usr/bin/php'):
+    report_info("Copying PHP related files...")
+    run.copy_file('/etc/php.ini', fail_ok=True)
+    if os.path.exists('/etc/php.d'):
+        run.copy_dir('/etc/php.d/', fail_ok=False, sort=False)
+
+    if os.path.exists('/etc/php/'):
+        run.copy_dir('/etc/php/', fail_ok=False, sort=False)
+
+    run.run_command(
+        ['php', '-m'], 'php_modules', fail_ok=False, sort=False)
+    run.run_command(
+        ['php', '-i'], 'php_info', fail_ok=False, sort=False)
+    if os.path.exists('/usr/bin/pecl'):
+        run.run_command(
+            ['pecl', 'list'], 'pecl-list', fail_ok=True, sort=False)
+
 
 # copy custom files and directories
 for section in Config.sections():
@@ -822,27 +822,8 @@ for section in Config.sections():
 
 report_info("\n")
 
-# php specifics
-if os.path.exists('/usr/bin/php'):
-    is_php_detected = True
-    report_info("Copying PHP related files...")
-    run.copy_file('/etc/php.ini', fail_ok=True)
-    if os.path.exists('/etc/php.d'):
-        run.copy_dir('/etc/php.d/', fail_ok=False, sort=False)
-
-    if os.path.exists('/etc/php/'):
-        run.copy_dir('/etc/php/', fail_ok=False, sort=False)
-
-    run.run_command(
-        ['php', '-m'], 'php_modules', fail_ok=False, sort=False)
-    run.run_command(
-        ['php', '-i'], 'php_info', fail_ok=False, sort=False)
-    if os.path.exists('/usr/bin/pecl'):
-        run.run_command(
-            ['pecl', 'list'], 'pecl-list', fail_ok=True, sort=False)
-
 if ('post' in phase or 'rollback' in phase or options.force_compare) and not options.silent_enabled:
-    compare_files()
+    compare_files(options)
 
 
 if options.archive_enabled:

configsnap.spec

@@ -1,12 +1,11 @@
 Name:          configsnap
-Version:       0.17.1
+Version:       0.18.0
 Release:       1%{?dist}
 Summary:       Record and compare system state
 License:       ASL 2.0
 URL:           https://github.com/rackerlabs/%{name}
 Source0:       https://github.com/rackerlabs/%{name}/archive/%{version}.tar.gz
 # Changes the python shebang to python2
-Patch0:        python_executable.patch
 BuildArch:     noarch
 BuildRequires: python2-devel
 BuildRequires: help2man
@@ -17,7 +16,6 @@ with a previous state and identify changes
 
 %prep
 %setup -q
-%patch0 -p0
 
 %build
 help2man --include=%{name}.help2man --no-info ./%{name} -o %{name}.man
@@ -42,6 +40,9 @@ install -p -m 0600 additional.conf %{buildroot}%{_sysconfdir}/%{name}/additional
 %{_sysconfdir}/%{name}
 
 %changelog
+* Mon Feb 03 2020 Nick Rhodes <[email protected]> - 0.18.0-1
+- Improvements to get_diff (PR 110)
+
 * Wed Jul 03 2019 Nick Rhodes <[email protected]> - 0.17.1-1
 - Convert relative basedir to absolute path (PR 103)