diff --git a/README.md b/README.md index 76b93b1..aff25cd 100644 --- a/README.md +++ b/README.md @@ -250,6 +250,48 @@ optional arguments: ``` +# push_puppetforge +This script allows users with an offline puppet-forge-server (rubygem) instance to +perform a special export of puppetforge modules from the Satellite puppet-forge +repository (-r) in the directory structure required by the puppet-forge-server +application. After exporting, the modules are copied via rsync to the puppet-forge-server. +The puppet-forge-server hostname can be defined in the config.yml, or overridden with +(-s), as can the module path (-m) on the remote server. The user performing the rsync +will be the user that is running the script, unless overridden with (-u). + +The config.yml block that defines the puppet-forge-server hostname is: +``` +puppet-forge-server: + hostname: puppetforge.example.org +``` + +``` +usage: push_puppetforge.py [-h] [-o ORG] [-r REPO] [-s SERVER] [-m MODULEPATH] [-u USER] + +Cleans content views for specified organization. + +optional arguments: + -h, --help show this help message and exit + -o ORG, --org ORG Organization (Uses default if not specified) + -r REPO, --repo REPO Puppetforge repository label + -s SERVER, --server SERVER + puppet-forge-server hostname + -m MODULEPATH, --modulepath MODULEPATH + path to puppet-forge-server modules + -u USER, --user USER Username to push modules to server as (default is user + running script) +``` + +### Examples +``` +./push_puppetforge.py -r Puppet_Forge +./push_puppetforge.py -r Puppet_Forge -u fred +./push_puppetforge.py -r Puppet_Forge -s test.example.org -m /opt/tmp +``` + + + + # clean_content_views This script removes orphaned versions of either all or nominated content views. This should be run periodically to clean out old/unused content view data from diff --git a/bin/push_puppetforge b/bin/push_puppetforge new file mode 100755 index 0000000..04a4e81 --- /dev/null +++ b/bin/push_puppetforge @@ -0,0 +1,11 @@ +#!/usr/bin/python +import sys + +sys.path.insert(0, '/usr/share/sat6_scripts') +try: + import push_puppetforge + push_puppetforge.main(sys.argv[1:]) +except KeyboardInterrupt, e: + print >> sys.stderr, "\n\nExiting on user cancel." + sys.exit(1) + diff --git a/config/config.yml.example b/config/config.yml.example index 6d7daaf..d50b4e8 100644 --- a/config/config.yml.example +++ b/config/config.yml.example @@ -40,3 +40,7 @@ cleanup: keep: 1 - view: RHEL Workstation keep: 3 + +puppet-forge-server: + hostname: puppetforge.example.org + diff --git a/helpers.py b/helpers.py index 512b352..90ee313 100644 --- a/helpers.py +++ b/helpers.py @@ -48,6 +48,8 @@ SYNCBATCH = CONFIG['import']['syncbatch'] else: SYNCBATCH = 255 +if 'hostname' in CONFIG['puppet-forge-server']: + PFSERVER = CONFIG['puppet-forge-server']['hostname'] # 'Global' Satellite 6 parameters # Satellite API diff --git a/push_puppetforge.py b/push_puppetforge.py new file mode 100755 index 0000000..2d4a269 --- /dev/null +++ b/push_puppetforge.py @@ -0,0 +1,223 @@ +#!/usr/bin/python +#title :sat_export.py +#description :Exports Satellite 6 Content for disconnected environments +#URL :https://github.com/RedHatSatellite/sat6_disconnected_tools +#author :Geoff Gatward +#notes :This script is NOT SUPPORTED by Red Hat Global Support Services. +#license :GPLv3 +#============================================================================== +""" +Exports Satellite 6 yum content. +""" + +import sys, argparse, datetime, os, shutil, pickle, re +import fnmatch, subprocess, tarfile +import simplejson as json +from glob import glob +import helpers + +try: + import yaml +except ImportError: + print "Please install the PyYAML module." + sys.exit(-1) + + +def export_puppet(repo_id, repo_label, repo_relative, export_type): + """ + Export Puppet modules + Takes the type (full/incr) + """ + numfiles = 0 + PUPEXPORTDIR = helpers.EXPORTDIR + '/puppet' + if not os.path.exists(PUPEXPORTDIR): + os.makedirs(PUPEXPORTDIR) + + msg = "Exporting Puppet repository id " + str(repo_id) + helpers.log_msg(msg, 'INFO') + + # This will currently export ALL ISO, not just the selected repo + msg = "Exporting all Puppet content" + helpers.log_msg(msg, 'INFO') + + msg = " Copying files for export..." + colx = "{:<70}".format(msg) + print colx[:70], + helpers.log_msg(msg, 'INFO') + # Force the status message to be shown to the user + sys.stdout.flush() + + os.system('find -L /var/lib/pulp/published/puppet/http/repos/*' + repo_label \ + + ' -type f -exec cp --parents -Lrp {} ' + PUPEXPORTDIR + ' \;') + + # At this point the puppet/ export dir will contain individual repos - we need to 'normalise' them + for dirpath, subdirs, files in os.walk(PUPEXPORTDIR): + for tdir in subdirs: + if repo_label in tdir: + # This is where the exported ISOs for our repo are located + INDIR = os.path.join(dirpath, tdir) + # And this is where we want them to be moved to so we can export them in Satellite format + # We need to knock off '/Library/' from beginning of repo_relative and replace with export/ + exportpath = "/".join(repo_relative.strip("/").split('/')[2:]) + OUTDIR = helpers.EXPORTDIR + '/export/' + exportpath + + # Move the files into the final export tree + if not os.path.exists(OUTDIR): + shutil.move(INDIR, OUTDIR) + + os.chdir(OUTDIR) + numfiles = sum([len(files) for r, d, files in os.walk(OUTDIR)]) + # Subtract the manifest from the number of files: + numfiles = numfiles - 1 + + # Since we are dealing with Puppet_Forge, create a second bundle for import to puppet-forge-server + if 'Puppet_Forge' in OUTDIR: + PFEXPORTDIR = helpers.EXPORTDIR + '/puppetforge' + if not os.path.exists(PFEXPORTDIR): + os.makedirs(PFEXPORTDIR) + os.system('find ' + OUTDIR + ' -name "*.gz" -exec cp {} ' + PFEXPORTDIR + ' \;') + + msg = 'Puppet Export OK (' + str(numfiles) + ' files)' + helpers.log_msg(msg, 'INFO') + print helpers.GREEN + msg + helpers.ENDC + + # Now we are done with the original export dumps, we can delete them. + shutil.rmtree(helpers.EXPORTDIR + '/export') + shutil.rmtree(helpers.EXPORTDIR + '/puppet') + + return numfiles + + +def copy_to_pfserver(export_dir, pfserver, pfmodpath, pfuser): + """ + Use rsync to copy the exported module tree to the puppet-forge-server instance + """ + target = pfuser + '@' + pfserver + ':' + pfmodpath + msg = 'Copying puppet modules to ' + target + '\n' + helpers.log_msg(msg, 'INFO') + print msg + os.system('rsync -avrzc ' + export_dir + '/* ' + target) + + +def main(args): + """ + Main Routine + """ + #pylint: disable-msg=R0912,R0914,R0915 + + # Who is running this script? + runuser = helpers.who_is_running() + + # Set the base dir of the script and where the var data is + global dir + global vardir + dir = os.path.dirname(__file__) + vardir = os.path.join(dir, 'var') + confdir = os.path.join(dir, 'config') + + # Check for sane input + parser = argparse.ArgumentParser(description='Performs Export of Default Content View.') + # pylint: disable=bad-continuation + parser.add_argument('-o', '--org', help='Organization (Uses default if not specified)', + required=False) + parser.add_argument('-r', '--repo', help='Puppetforge repo label', required=False) + parser.add_argument('-s', '--server', help='puppet-forge-server hostname', required=False) + parser.add_argument('-m', '--modulepath', help='path to puppet-forge-server modules', + required=False) + parser.add_argument('-u', '--user', help='Username to push modules to server as (default is user running script)', + required=False) + args = parser.parse_args() + + # Set our script variables from the input args + if args.org: + org_name = args.org + else: + org_name = helpers.ORG_NAME + + # Define the puppet-forge-server hostname + if args.server: + pfserver = args.server + else: + if not helpers.PFSERVER: + print "Puppet forge server not defined" + sys.exit(-1) + else: + pfserver = helpers.PFSERVER + + # Set the remote (puppet-forge-server) modules directory + if args.modulepath: + modpath = args.modulepath + else: + modpath = '/opt/puppetforge/modules' + + # Set the username to use to push modules + if args.user: + pfuser = args.user + else: + pfuser = runuser + + # Record where we are running from + script_dir = str(os.getcwd()) + + # Get the org_id (Validates our connection to the API) + org_id = helpers.get_org_id(org_name) + + # Read the repo label given by the user + if args.repo: + pfrepo = args.repo + else: + print "Puppetforge repo not defined" + sys.exit(-1) + + # Remove any previous exported content left behind by prior unclean exit + if os.path.exists(helpers.EXPORTDIR + '/export'): + msg = "Removing existing export directory" + helpers.log_msg(msg, 'DEBUG') + shutil.rmtree(helpers.EXPORTDIR + '/export') + + # Collect a list of enabled repositories. This is needed for: + # 1. Matching specific repo exports, and + # 2. Running import sync per repo on the disconnected side + repolist = helpers.get_p_json( + helpers.KATELLO_API + "/repositories/", \ + json.dumps( + { + "organization_id": org_id, + "per_page": '1000', + } + )) + + + # Process each repo + for repo_result in repolist['results']: + if repo_result['content_type'] == 'puppet': + # If we have a match, do the export + if repo_result['label'] == pfrepo: + + # Trigger export on the repo + numfiles = export_puppet(repo_result['id'], repo_result['label'], repo_result['relative_path'], 'full') + + else: + msg = "Skipping " + repo_result['label'] + helpers.log_msg(msg, 'DEBUG') + + + # Now we need to process the on-disk export data. + # Define the location of our exported data. + export_dir = helpers.EXPORTDIR + "/puppetforge" + + # Now we can copy the content to the puppet-forge-server instance + os.chdir(script_dir) + copy_to_pfserver(export_dir, pfserver, modpath, pfuser) + + + # And we're done! + print helpers.GREEN + "Puppet Forge export complete.\n" + helpers.ENDC + + +if __name__ == "__main__": + try: + main(sys.argv[1:]) + except KeyboardInterrupt, e: + print >> sys.stderr, ("\n\nExiting on user cancel.") + sys.exit(1) diff --git a/push_puppetforge.sh b/push_puppetforge.sh deleted file mode 100755 index ddd3249..0000000 --- a/push_puppetforge.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -# Placeholder script until python version written - -# requires satellite root ssh pubkey pushed to forge@puppet-forge-server - - -PFSERVER=$1 - -if [ $# -ne 1 ]; then - echo "Please specify the puppet-forge server hostname" - exit 1 -fi - -scp /var/sat-content/puppetforge/*.gz forge@$PFSERVER:/opt/puppetforge/modules -#ssh forge@$PFSERVER chown -R forge:forge /opt/puppetforge/modules -#ssh forge@$PFSERVER restorecon -R /opt/puppetforge/modules - diff --git a/sat6_scripts.spec b/sat6_scripts.spec deleted file mode 100644 index e686028..0000000 --- a/sat6_scripts.spec +++ /dev/null @@ -1,99 +0,0 @@ -Name: sat6_scripts -Version: 0.1 -Release: 3%{?dist} -Summary: Scripts to automate Satellite 6 tasks - -License: GPL -URL: https://github.com/ggatward/sat6_scripts -Source0: sat6_scripts-0.1.tar.gz - -Requires: python >= 2.7, PyYAML - -%description -Various scripts to substantially automate management tasks of Satellite 6, including: -- content export/import in disconnected environments, -- content publish/promote activities - - -%prep -%autosetup - - -%build - - -%install -rm -rf %{buildroot} -mkdir -p %{buildroot}/usr/local/{bin,etc/sat6_scripts} -mkdir -p %{buildroot}/usr/share/{doc/sat6_scripts,sat6_scripts/config} -install -m 0644 README.md %{buildroot}/usr/share/doc/sat6_scripts/README.md -install -m 0644 LICENSE %{buildroot}/usr/share/doc/sat6_scripts/LICENSE -install -m 0644 docs/sat62_install.txt %{buildroot}/usr/share/doc/sat6_scripts/sat62_install.txt -install -m 0644 docs/sat62disc_install.txt %{buildroot}/usr/share/doc/sat6_scripts/sat62disc_install.txt -install -m 0644 docs/sat62_hardening.txt %{buildroot}/usr/share/doc/sat6_scripts/sat62_hardening.txt -install -m 0644 config/config.yml.example %{buildroot}/usr/share/sat6_scripts/config/config.yml -install -m 0644 config/exports.yml.example %{buildroot}/usr/share/sat6_scripts/config/exports.yml -install -m 0755 bin/check_sync %{buildroot}/usr/local/bin/check_sync -install -m 0755 bin/sat_export %{buildroot}/usr/local/bin/sat_export -install -m 0755 bin/sat_import %{buildroot}/usr/local/bin/sat_import -install -m 0755 bin/clean_content_views %{buildroot}/usr/local/bin/clean_content_views -install -m 0755 bin/publish_content_views %{buildroot}/usr/local/bin/publish_content_views -install -m 0755 bin/promote_content_views %{buildroot}/usr/local/bin/promote_content_views -install -m 0755 bin/download_manifest %{buildroot}/usr/local/bin/download_manifest -install -m 0644 helpers.py %{buildroot}/usr/share/sat6_scripts/helpers.py -install -m 0644 check_sync.py %{buildroot}/usr/share/sat6_scripts/check_sync.py -install -m 0644 sat_export.py %{buildroot}/usr/share/sat6_scripts/sat_export.py -install -m 0644 sat_import.py %{buildroot}/usr/share/sat6_scripts/sat_import.py -install -m 0644 publish_content_views.py %{buildroot}/usr/share/sat6_scripts/publish_content_views.py -install -m 0644 promote_content_views.py %{buildroot}/usr/share/sat6_scripts/promote_content_views.py -install -m 0644 clean_content_views.py %{buildroot}/usr/share/sat6_scripts/clean_content_views.py -install -m 0644 download_manifest.py %{buildroot}/usr/share/sat6_scripts/download_manifest.py - - - -%files -%doc /usr/share/doc/sat6_scripts/README.md -%doc /usr/share/doc/sat6_scripts/sat62_install.txt -%doc /usr/share/doc/sat6_scripts/sat62disc_install.txt -%doc /usr/share/doc/sat6_scripts/sat62_hardening.txt -%license /usr/share/doc/sat6_scripts/LICENSE -%config(noreplace) /usr/share/sat6_scripts/config/config.yml -%config(noreplace) /usr/share/sat6_scripts/config/exports.yml - -/usr/share/sat6_scripts/helpers.py -/usr/share/sat6_scripts/check_sync.py -/usr/share/sat6_scripts/sat_export.py -/usr/share/sat6_scripts/sat_import.py -/usr/share/sat6_scripts/publish_content_views.py -/usr/share/sat6_scripts/promote_content_views.py -/usr/share/sat6_scripts/clean_content_views.py -/usr/share/sat6_scripts/download_manifest.py - -/usr/local/bin/check_sync -/usr/local/bin/sat_export -/usr/local/bin/sat_import -/usr/local/bin/clean_content_views -/usr/local/bin/publish_content_views -/usr/local/bin/promote_content_views -/usr/local/bin/download_manifest - -%exclude /usr/share/sat6_scripts/*.pyc -%exclude /usr/share/sat6_scripts/*.pyo - -%post -# Only run on initial install -if [ $1 -eq 1 ]; then - ln -s /usr/share/sat6_scripts/config/config.yml /usr/local/etc/sat6_scripts/config.yml - ln -s /usr/share/sat6_scripts/config/exports.yml /usr/local/etc/sat6_scripts/exports.yml -fi - -%postun -# Only run on a complete uninstall -if [ $1 -eq 0 ]; then - unlink /usr/local/etc/sat6_scripts/config.yml - unlink /usr/local/etc/sat6_scripts/exports.yml -fi - - -%changelog - diff --git a/sat_export.py b/sat_export.py index 352548b..22a1176 100755 --- a/sat_export.py +++ b/sat_export.py @@ -688,7 +688,7 @@ def main(args): helpers.log_msg(msg, 'ERROR') sys.exit(-1) - msg = "Specific environment export called for " + ename + ". Configured repos:" + msg = "Specific environment export called for " + ename + "." helpers.log_msg(msg, 'DEBUG') for repo in erepos: msg = " - " + repo @@ -703,6 +703,8 @@ def main(args): # Log the fact we are starting msg = "------------- Content export started by " + runuser + " ----------------" if not args.last: + if args.env: + msg = "------ " + ename + " Content export started by " + runuser + " ---------" helpers.log_msg(msg, 'INFO') # Get the current time - this will be the 'last export' time if the export is OK @@ -768,6 +770,9 @@ def main(args): last_export = export_times['DoV'] if since: last_export = since_export + else: + # To ensure we get ALL the packages reset the time to midnight on the last_export day + last_export = last_export.split(' ')[0] + " 00:00:00" colb = "(INCR since " + last_export + ")" else: export_type = 'full' @@ -835,6 +840,9 @@ def main(args): last_export = export_times[repo_result['label']] if since: last_export = since_export + else: + # To ensure we get ALL the packages reset the time to midnight on the last_export day + last_export = last_export.split(' ')[0] + " 00:00:00" colb = "(INCR since " + last_export + ")" else: export_type = 'full' @@ -914,6 +922,9 @@ def main(args): last_export = export_times[repo_result['label']] if since: last_export = since_export + else: + # To ensure we get ALL the packages reset the time to midnight on the last_export day + last_export = last_export.split(' ')[0] + " 00:00:00" colb = "(INCR since " + last_export + ")" else: export_type = 'full' @@ -960,6 +971,9 @@ def main(args): last_export = export_times[repo_result['label']] if since: last_export = since_export + else: + # To ensure we get ALL the packages reset the time to midnight on the last_export day + last_export = last_export.split(' ')[0] + " 00:00:00" colb = "(INCR since " + last_export + ")" else: export_type = 'full'