Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

New cmp fix #110

Merged
merged 6 commits into from
Feb 3, 2020
Merged

New cmp fix #110

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
aa9451e
Move running process check into dataGather method
rhodesn Jan 19, 2020
cfadd93
Resolve flake8 test
rhodesn Jan 19, 2020
abcdbfa
changing paradigm to select which files to compare, should allow bett…
Jan 21, 2020
8e44a17
ready for review
Jan 21, 2020
7c61766
flake8 compliant
Jan 21, 2020
c0f6350
handling comparison exclusions
Jan 22, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
253 changes: 117 additions & 136 deletions configsnap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,8 @@ import glob
import re
import tarfile


version = "0.17.1"
diffs_found_msg = False
is_php_detected = False
is_apache_detected = False
diffs_found = False


def report_verbose(message):
Expand Down Expand Up @@ -128,98 +125,73 @@ def create_archive(tagdir, workdir, tag, overwrite):
return output_filename, True


def compare_files():
# Check that there are pre files, and exit if there aren't
found_pre = False
def compare_files(options):
"""Compare 2 phases of a same tag"""
run = dataGather(workdir, phase)

for filename in os.listdir(workdir):
if filename.endswith(options.pre_suffix):
found_pre = True
if not found_pre:
report_error("Unable to diff, as no " +
options.pre_suffix + " pre files")
sys.exit(0)

report_info("\nThis is %s phase so performing file diffs..." % phase)

if 'rollback' in phase:
run.get_diff_files('packages')

compare_files = ['sysvinit', 'mounts', 'netstat', 'ip_addresses',
'network', 'interfaces', 'hosts', 'lvs', 'pvs', 'vgs',
'fstab', 'powermt', 'omreport_version', 'ip_routes',
'partitions', 'lspci', 'omreport_memory',
'omreport_processors', 'omreport_nics', 'omreport_vdisk',
'omreport_pdisk', 'hpreport_server', 'hpreport_memory',
'hpreport_bios', 'hpreport_storage_controller',
'hpreport_storage_vdisk', 'hpreport_storage_pdisk',
'multipath', 'systemdinit', 'upstartinit', 'yum.conf',
'dnf.conf', 'pcs_constraints', 'sudoers']

# Add php files if detected
if is_php_detected:
compare_files.append('php.ini')
if os.path.exists('/usr/bin/pecl'):
compare_files.append('pecl-list')

# Add apache files if detected
if is_apache_detected:
compare_files.append('httpd_modules')
compare_files.append('httpd_vhosts')

# Array to hold custom collection files and commands to compare
additional_to_compare = []
for section in Config.sections():
try:
if Config.has_option(section, 'compare') and Config.get(section, 'compare').lower() == "true":
additional_to_compare.append(section)
except ConfigParser.NoOptionError, e:
report_error(
"Check config file options for section %s: %s" % (section, e))
except Exception, e:
report_error(
"Could not parse config file section %s: %s" % (section, e))
# Collects all files and dir at the top level
all_files = os.listdir(workdir)
# Those will always result in positive diffs, this is misleading so we exclude them
exclusions = '^(sysctl|vgcfgbackup[-].*|ps|meminfo|dmesg|crm_mon|pcs_status)$'

compare_files.extend(additional_to_compare)
# Proceed if at least one file with options.pre_suffix is present
if any(filename.endswith(".%s" % options.pre_suffix) for filename in all_files):
report_info("Perfoming file diffs...\n")

for filename in compare_files:
pre_filename = os.path.join(workdir,
"%s.%s" % (filename, options.pre_suffix))
if os.path.exists(pre_filename):
run.get_diff_files(filename)
if 'rollback' in phase:
run.get_diff_files('packages', options)

# Report subdirectories at the subdir level unless ther verbose option is
# given, in which case report as usual
report_info("\nStarting sub-dir diffs...")
for directory in os.listdir(workdir):
if os.path.isdir(os.path.join(workdir, directory)):
run.get_diff_dir(directory)
# to search for files with pre_suffix or phase only
suffix_exist = re.compile('[.](' + options.pre_suffix + '|' + options.phase + ')$')
filtered_files = []
for index, value in enumerate(all_files):
if suffix_exist.search(value):
filebase = suffix_exist.sub('', value)
if not re.match(exclusions, filebase):
filtered_files.append(filebase)

try:
uname_pre_fd = open(
os.path.join(workdir, 'uname.' + options.pre_suffix), 'r')
uname_post_fd = open(
os.path.join(workdir, "uname.%s" % phase), 'r')
uname_pre = uname_pre_fd.read().strip()
uname_post = uname_post_fd.read().strip()
if uname_pre != uname_post:
sys.stdout.write("Old uname: %s. New uname: %s\n" %
(uname_pre, uname_post))
except Exception, e:
report_error(
"Unable to open file %s: %s" % ((e.filename, e.strerror)))
compare_files = list(set(filtered_files))
compare_files.sort()

if diffs_found_msg:
report_info_blue("\nINTERPRETING DIFFS:\n")
report_info_blue("* Lines beginning with a - show an "
"entry from the " + options.pre_suffix + " pre file "
"which has changed\n" "or which is not present in the .post or .rollback file.\n")
report_info_blue("* Lines beginning with a + show an entry from the "
".post or.rollback file which\nhas changed or which "
"is not present in the " + options.pre_suffix + " pre file.\n")
report_info_blue("Please review all diff output above carefully and "
"account for any differences\nfound.\n")
for filename in compare_files:
run.get_diff_files(filename, options)

# Report subdirectories at the subdir level unless ther verbose option is
# given, in which case report as usual
report_info("\nStarting sub-dir diffs...")
for directory in os.listdir(workdir):
if os.path.isdir(os.path.join(workdir, directory)):
run.get_diff_dir(directory, options)

try:
uname_pre_fd = open(
os.path.join(workdir, 'uname.' + options.pre_suffix), 'r')
uname_post_fd = open(
os.path.join(workdir, "uname.%s" % phase), 'r')
uname_pre = uname_pre_fd.read().strip()
uname_post = uname_post_fd.read().strip()
if uname_pre != uname_post:
sys.stdout.write("Old uname: %s. New uname: %s\n" %
(uname_pre, uname_post))
except Exception, e:
report_error(
"Unable to open file %s: %s" % ((e.filename, e.strerror)))

if diffs_found:
report_info_blue("\nINTERPRETING DIFFS:\n")
report_info_blue("* Lines beginning with a - show an "
"entry from the " + options.pre_suffix + " pre file "
"which has changed\n" "or which is not present in the .post or .rollback file.\n")
report_info_blue("* Lines beginning with a + show an entry from the "
".post or.rollback file which\nhas changed or which "
"is not present in the " + options.pre_suffix + " pre file.\n")
report_info_blue("Please review all diff output above carefully and "
"account for any difference found.\n")

else:
report_error("Unable to diff, no files with suffix \"" +
options.pre_suffix + "\" present.")
sys.exit(1)


class dataGather:
Expand Down Expand Up @@ -327,8 +299,7 @@ class dataGather:
report_error("Unable to open %s: %s" % (filename, e.strerror))
return False

def get_diff_files(self, filename):
global diffs_found_msg
def get_diff_files(self, filename, options):
filename = os.path.join(self.workdir, filename)
pre_filename = "%s.%s" % (filename, options.pre_suffix)
post_filename = "%s.%s" % (filename, self.phase)
Expand All @@ -350,16 +321,19 @@ class dataGather:
diff_stdout, diff_stderr = diff_proc.communicate(input=None)

if diff_proc.returncode != 0:
diffs_found_msg = True
global diffs_found
diffs_found = True
report_error(
"Differences found against %s.%s:\n" % (filename, options.pre_suffix))
sys.stdout.writelines(diff_stdout)
if diff_stdout != "":
sys.stdout.writelines(diff_stdout)
else:
report_error(diff_stderr)
else:
report_info("No differences against %s.%s" %
(filename, options.pre_suffix))

def get_diff_dir(self, sdir):
diffs_found_msg = False
def get_diff_dir(self, sdir, options):
all_pre_files = []
sdir = os.path.join(workdir, sdir)

Expand Down Expand Up @@ -393,24 +367,46 @@ class dataGather:
return None

if diff_proc.wait() != 0:
diffs_found_msg = True
global diffs_found
diffs_found = True
report_error("Differences found against %s:\n" % pre_filename)
sys.stdout.writelines(diff_proc.stdout.readlines())
elif options.verbose_enabled is True:
report_verbose("No differences against %s" % pre_filename)

# Looking for new files
local_diffs = False
for post_filename in glob.glob(sdir + '/*' + self.phase):
if post_filename not in all_pre_files:
report_error("+ File added: %s:" % post_filename)
diffs_found_msg = True
local_diffs = True

if local_diffs is False:
report_info("No extra post files found in %s" % sdir)

def is_running(self, *paths):
if len(self.procs_out) == 0:
# Get a list of processes running on the server
procs = subprocess.Popen(
['ps', '-A', '--noheaders', '-o', 'args'], stdout=subprocess.PIPE,
stderr=subprocess.PIPE)

procs_out, procs_err = procs.communicate()
if procs.returncode == 0:
self.procs_out = [x.split()[0] for x in procs_out.splitlines()]
else:
report_error('Failed to get process list')

for path in paths:
if path in self.procs_out:
return True

if diffs_found_msg is False:
report_info("No differences found for files in %s" % sdir)
return False

def __init__(self, workdir, phase):
self.workdir = workdir
self.phase = phase
self.procs_out = []


###################################################
Expand Down Expand Up @@ -538,7 +534,7 @@ else:
if options.compare_only:
report_info("Comparing files from phases: %s and %s" %
(options.pre_suffix, options.phase))
compare_files()
compare_files(options)
sys.exit(0)

# Ensure each section has a valid Type
Expand All @@ -548,19 +544,6 @@ for section in Config.sections():
report_error("Wrong \"Type\" defined for custom collection entry \"%s\"; "
"should be \"file\", \"directory\" or \"command\"" % section)

# Get a list of processes running on the server
procs = subprocess.Popen(
['ps', '-A', '--noheaders', '-o', 'args'], stdout=subprocess.PIPE,
stderr=subprocess.PIPE)

procs_output = procs.communicate()[0]
if procs.returncode != 0:
report_error('Failed to get process list')

# Just keep the process name, drop the arguments
procs_output = procs_output.splitlines()
procs_output = [x.split()[0] for x in procs_output]

run = dataGather(workdir, phase)
report_info("Getting storage details (LVM, partitions, multipathing)...")
if os.path.exists('/sbin/lvs'):
Expand Down Expand Up @@ -717,15 +700,14 @@ if os.path.exists(hpasmpath):
break
break

if '/usr/sbin/httpd' in procs_output or '/usr/sbin/httpd.worker' in procs_output:
if run.is_running('/usr/sbin/httpd', '/usr/sbin/httpd.worker'):
report_info("Getting Apache vhosts and modules...")
is_apache_detected = True
run.run_command(
['httpd', '-S'], 'httpd_vhosts', fail_ok=False, sort=False)
run.run_command(
['httpd', '-M'], 'httpd_modules', fail_ok=False, sort=True)

if '/usr/libexec/mysqld' in procs_output:
if run.is_running('/usr/libexec/mysqld', '/usr/sbin/mysqld'):
report_info("Getting MySQL databases...")
run.run_command(['mysql', '-Bse', 'show databases;'],
'mysql_databases', fail_ok=True, sort=True)
Expand Down Expand Up @@ -785,6 +767,24 @@ if os.path.exists('/etc/network/interfaces.d'):
if os.path.exists('/etc/sysconfig/network-scripts/'):
run.copy_dir('/etc/sysconfig/network-scripts/', file_pattern='(ifcfg-|route-).*',
fail_ok=False, sort=False)
# php specifics
if os.path.exists('/usr/bin/php'):
report_info("Copying PHP related files...")
run.copy_file('/etc/php.ini', fail_ok=True)
if os.path.exists('/etc/php.d'):
run.copy_dir('/etc/php.d/', fail_ok=False, sort=False)

if os.path.exists('/etc/php/'):
run.copy_dir('/etc/php/', fail_ok=False, sort=False)

run.run_command(
['php', '-m'], 'php_modules', fail_ok=False, sort=False)
run.run_command(
['php', '-i'], 'php_info', fail_ok=False, sort=False)
if os.path.exists('/usr/bin/pecl'):
run.run_command(
['pecl', 'list'], 'pecl-list', fail_ok=True, sort=False)


# copy custom files and directories
for section in Config.sections():
Expand Down Expand Up @@ -822,27 +822,8 @@ for section in Config.sections():

report_info("\n")

# php specifics
if os.path.exists('/usr/bin/php'):
is_php_detected = True
report_info("Copying PHP related files...")
run.copy_file('/etc/php.ini', fail_ok=True)
if os.path.exists('/etc/php.d'):
run.copy_dir('/etc/php.d/', fail_ok=False, sort=False)

if os.path.exists('/etc/php/'):
run.copy_dir('/etc/php/', fail_ok=False, sort=False)

run.run_command(
['php', '-m'], 'php_modules', fail_ok=False, sort=False)
run.run_command(
['php', '-i'], 'php_info', fail_ok=False, sort=False)
if os.path.exists('/usr/bin/pecl'):
run.run_command(
['pecl', 'list'], 'pecl-list', fail_ok=True, sort=False)

if ('post' in phase or 'rollback' in phase or options.force_compare) and not options.silent_enabled:
compare_files()
compare_files(options)


if options.archive_enabled:
Expand Down