Migrate scripts from geni-portal

CHANGES

@@ -31,6 +31,7 @@
    It also looks at the older `chapi.log.1` and `error.log.1`. (#360)
  * `geni-check-errors` now also ignores collector tool speaksfor. (#361)
  * Add iMinds w-iLab.t and Virtual Wall 1 aggregates (#367)
+ * Migrate management scripts from geni-portal to geni-ch (#101)
  * Add Kaiserslautern OpenGENI aggreate (#374)
 
 == 1.28 ==

bin/Makefile.am

@@ -61,14 +61,35 @@ dist_pkgdata_DATA = \
 	ch-error-log-out.txt \
 	chapi-log-out.txt
 
+dist_bin_SCRIPTS = \
+                geni-add-member-attribute \
+                geni-add-project-member \
+                geni-add-trusted-tool \
+                geni-assert-email \
+                geni-delete-outside-cert \
+                geni-disable-user \
+                geni-enable-user \
+	        geni-ops-report \
+                geni-remove-member-attribute \
+                geni-remove-project-member \
+                gmoc_list_contacts \
+                gmoc_list_slices \
+                report_genich_relations
+
 dist_sbin_SCRIPTS = \
-	geni-add-member-privilege \
 	geni-create-ma-crl \
+	geni-init-ca \
+	geni-init-services \
 	geni-list-member-projects \
 	geni-list-pending-requests \
 	geni-revoke-member-certificate \
 	geni-revoke-member-privilege
 
+if INSTALL_GITHASH
+dist_pkgsysconf_DATA = geni-ch-githash
+endif
+
+
 # GPO Lab
 # Install in /usr/local/lib, not /usr/lib
 localpythondir = $(subst /usr/lib,/usr/local/lib,$(pythondir))

bin/archive_log_entry_tables.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# Moved expired data from logging_entry to logging_entry_old
+# Moved associated data from logging_entry_attribute to logging_entry_attribute_old
+#
+# Usage: archive_log_entry_table.sh expired_time
+
+if [ $# -ne 1 ]; then
+    echo "Usage: archive_log_entry_tables.sh time (ex. '2014-03-10')"
+    exit
+else
+
+    EXPIRED_TIME=$1
+    echo "BEGIN;" > /tmp/log_entry_mod.sql
+    echo "insert into logging_entry_old (select * from logging_entry where event_time < '$EXPIRED_TIME');" >> /tmp/log_entry_mod.sql
+    echo "insert into logging_entry_attribute_old select * from logging_entry_attribute where event_id in (select id from logging_entry where event_time < '$EXPIRED_TIME');" >> /tmp/log_entry_mod.sql
+    echo "delete from logging_entry_attribute where event_id in (select id from logging_entry where event_time < '$EXPIRED_TIME');" >> /tmp/log_entry_mod.sql
+    echo "delete from logging_entry where event_time < '$EXPIRED_TIME';" >> /tmp/log_entry_mod.sql
+    echo "COMMIT;" >> /tmp/log_entry_mod.sql
+    psql -U portal -h localhost portal -f /tmp/log_entry_mod.sql
+fi

bin/archive_sa_entry_tables.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Moved expired data from logging_entry to logging_entry_old
+# Moved associated data from logging_entry_attribute to logging_entry_attribute_old
+#
+# Usage: archive_sa_entry_table.sh expired_time
+
+if [ $# -ne 1 ]; then
+    echo "Usage: archive_sa_entry_tables.sh time (ex. '2014-03-10')"
+    exit
+else
+    EXPIRED_TIME=$1
+    echo "BEGIN;" > /tmp/slice_tables_mod.sql
+    echo "insert into sa_slice_member_old (select * from sa_slice_member where slice_id in (select slice_id from sa_slice where expiration < '$EXPIRED_TIME')); " >> /tmp/slice_tables_mod.sql
+    echo "insert into sa_slice_old (select * from sa_slice where expiration < '$EXPIRED_TIME'); " >> /tmp/slice_tables_mod.sql
+    echo "delete from sa_slice_member where slice_id in (select slice_id from sa_slice where expiration < '$EXPIRED_TIME'); " >> /tmp/slice_tables_mod.sql
+    echo "delete from sa_slice where expiration < '$EXPIRED_TIME'; " >> /tmp/slice_tables_mod.sql
+    echo "COMMIT;" >> /tmp/slice_tables_mod.sql
+    psql -U portal -h localhost portal -f /tmp/slice_tables_mod.sql
+fi

bin/check_existence_of_accounts

@@ -0,0 +1,202 @@
+#!/usr/bin/env python
+# -*- mode:python -*-
+
+#----------------------------------------------------------------------
+# Copyright (c) 2013-2015 Raytheon BBN Technologies
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and/or hardware specification (the "Work") to
+# deal in the Work without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Work, and to permit persons to whom the Work
+# is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Work.
+#
+# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS
+# IN THE WORK.
+#----------------------------------------------------------------------
+
+import csv
+import optparse
+import sys
+
+check_list_file_default = 'tutorial.csv'
+accounts_list_file_default = 'accounts.csv'
+output_list_file_default = 'output.csv'
+
+check_fieldnames = ['email', 'first_name', 'last_name']
+accounts_fieldnames = ['contact_urn','contact_email','contact_lastname','contact_givenname','owner_id']
+output_fieldnames = ['email', 'name']
+
+class Person( object ):
+      def __init__( self, first_name=None, last_name=None, email=None ):
+      	  self.first_name = first_name	 
+	  self.last_name = last_name
+	  self.email = email
+      def __str__( self ):
+      	  return "%s %s (%s)" % (self.first_name, self.last_name, self.email)
+
+def main():
+      usage = "%prog --check-file tutorial.csv \\\n" \
+          "\t\t --accounts-file accounts.csv " \
+          "--output-file output.csv\n\n" \
+          "Arguments:\n" \
+          "\t--check-file is a CSV of the accounts to check for.  Each line is of "\
+          "\t\t\tthe form: \n" \
+          "\t\t\temail, first name, last name\n" \
+          "\t--accounts-file is the output of running: \n" \
+          "\t\tsudo -u www-data /usr/local/bin/geni-ops-report -u portal -P /usr/sysadmin/etc/portal_password -g > accounts.csv\n" \
+          "\t--output-file contains three parts:\n" \
+          "\t\t* People who HAVE accounts reported in the bulk upload format: \n"\
+          "\t\t\temail, first_name last_name\n" \
+          "\t\t* People who DO NOT have accounts reported in the same format \n"\
+          "\t\t  as --check-file\n" \
+          "\t\t* People who MAY have accounts and possible matches: \n" \
+          "\t\t\t-People being looked up are listed in the same format \n"\
+          "\t\t\t as --check-file\n" \
+          "\t\t\t-Accounts which may be a match are listed in the bulk \n"\
+          "\t\t\t upload format\n" \
+          "\nThe idea is that a file containing people for whom their was no account can be"\
+          "\npassed back into the script and compared to an updated --accounts-file at a "\
+          "\nlater time."
+      parser = optparse.OptionParser( usage=usage )
+      parser.add_option("-c", "--check-file",
+                            help="File containing a list of people to check if they have accounts (in CSV)", metavar="FILE", default=check_list_file_default, dest='check_list_file')
+      parser.add_option("-a", "--accounts-file",
+                            help="File containing a list of accounts (in CSV)", metavar="FILE", default=accounts_list_file_default, dest='accounts_list_file')
+      parser.add_option("-o", "--output-file",
+                            help="File containing a list of users from check-file which have accounts.  Output in a CSV suitable for use with bulk_upload.", metavar="FILE", default=output_list_file_default, dest='output_list_file')
+      parser.add_option("-q", "--quiet",
+                        help="Print less output", action='store_false', 
+                        default=True, dest='verbose')
+
+      if sys.argv is None:
+              # prints to stderr
+              parser.print_help()
+              exit(-1)
+      (options, args) = parser.parse_args(sys.argv)
+      check_list_file = options.check_list_file
+      accounts_list_file = options.accounts_list_file
+      output_list_file = options.output_list_file
+
+      # read the two input files
+      check_list = open(check_list_file)
+      check = csv.DictReader( check_list, fieldnames=check_fieldnames)
+      accounts_list = open(accounts_list_file)
+      accounts = csv.DictReader( accounts_list, fieldnames=accounts_fieldnames)
+
+      check_set = set() # the set of people who's accounts you are looking up
+      for row in check:
+          check_set.add(Person( row['first_name'].strip().lower(), row['last_name'].strip().lower(), email=row['email'].strip().lower() ))
+
+
+      accounts_set = set() # the set of people who have accounts
+      accounts_by_email = {} # dict keyed by email where value a member of the above set
+      accounts_by_lastname = {} # dict keyed by lastname where value a member of the above set
+      for row in accounts:
+          first_name = row['contact_givenname'].strip().lower()
+          last_name = row['contact_lastname'].strip().lower() 
+          email = email=row['contact_email'].strip().lower()
+          account_obj = Person( first_name, last_name, email=email)
+          accounts_set.add( account_obj )
+          accounts_by_email[email] = account_obj
+#          accounts_by_lastname[last_name] = account_obj
+          if not accounts_by_lastname.has_key(last_name):
+                accounts_by_lastname[ last_name ] = set()
+          accounts_by_lastname[ last_name ].add( account_obj )
+
+
+      output_set = set() # the set of people who we are looking up who HAVE accounts
+      output_options = {} # the set of people we are looking up who MAY have accounts
+      output_none = [] # the set of people we are looking who DO NOT have accounts (that we can find at least)
+      check_by_email = {} # dict keyed by email where value is a a member of check_set
+      for check_user in check_set:
+          check_by_email[check_user.email] = check_user
+          # People who HAVE accounts (verified by email)
+          if accounts_by_email.has_key(check_user.email):
+             account_obj = accounts_by_email[check_user.email]
+             print_obj = Person( check_user.first_name, check_user.last_name, account_obj.email )
+             output_set.add( print_obj )
+          elif accounts_by_lastname.has_key(check_user.last_name):
+             account_obj_list = accounts_by_lastname[check_user.last_name]
+             for account_obj in account_obj_list:
+                   if account_obj.first_name == check_user.first_name:       
+                         # People who HAVE have accounts (last name and first name match)
+                         print_obj = Person( check_user.first_name, check_user.last_name, account_obj.email )
+                         output_set.add( print_obj)
+                         if output_options.has_key(check_user.email):
+                               del output_options[ check_user.email ]
+                         break
+                   else:
+                      if not output_options.has_key(check_user.email):
+                         output_options[ check_user.email ] = set()
+                      # People who MAY have accounts (last name matches)
+                      output_options[ check_user.email ].add( account_obj )
+          else:
+              # People who DO NOT have accounts (none of email, last name, first name matches)
+              output_none.append(check_user.email)    
+      check_list.close()
+      accounts_list.close()
+
+      # write the output to files
+      with open(output_list_file, 'wb') as outfile:
+           writer = csv.writer(outfile)
+           # People who HAVE accounts (verified by email OR last name and first name match)
+           for person in output_set:
+                writer.writerow( [person.email, person.first_name+" "+person.last_name] )
+           # People who MAY have accounts (last name matches)
+           for key, output2 in output_options.items():
+                person2 = check_by_email[key]
+                if len(output2)>0:
+                      print >>outfile, "# No obvious entry found for..."
+                      writer.writerow( [key, person2.first_name, person2.last_name] )
+                      print >>outfile, "# These are possible fits for the above person:"
+                for person in output2:   
+                    writer.writerow( ["#"+person.email, person.first_name+" "+person.last_name] )
+           # People who DO NOT have accounts (none of email, last name, first name matches)
+           if len(output_none) > 0:
+                 print >>outfile, "# These people have no account:"
+           for output3 in output_none:
+                person3 = check_by_email[output3]
+                writer.writerow( [person3.email, person3.first_name, person3.last_name] )
+
+      # print the output to the screen
+      if options.verbose:
+            print "="*80
+            print " Names to check "
+            for person in check_set:
+                  print person
+
+            #print "="*80
+            #print " Accounts "
+            #for account in accounts_set:
+            #    print account
+
+            print "="*80
+            print " Output "
+            for output in output_set:
+                  print output
+            print "="*80
+            print " Output Options "
+            for key, output2 in output_options.items():
+                  person = check_by_email[key]
+                  if len(output2)>0:
+                        print "# Possible values for "+person.first_name+" "+person.last_name+" ("+key+"):"
+                  for person2 in output2:
+                        print person2
+            print "# These people have no account:"
+            for output3 in output_none:
+                  person3 = check_by_email[output3]
+                  print person3.first_name+" "+person3.last_name+" ("+person3.email+") "
+
+
+if __name__ == "__main__":
+    sys.exit(main())

bin/create_standard_services.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# To create service certs and SQL:
+# bin/geni-init-services --sql services.sql /etc/geni-ch/ch-services.ini
+
+# To add the services to the database (see above)
+# psql -U portal -h localhost -f services.sql portal
+
+# To add the portal:
+# geni-add-trusted-tool -d portal --host localhost -u portal portal \
+#                       urn:publicid:IDN+ch.geni.net+authority+portal
+
+
+echo
+echo "Warning: create_standard_services.sh is obsolete."
+echo
+echo "    Use 'geni-init-services' to create service certificates and SQL."
+echo "    Use 'geni-add-trusted-tool' to add the portal to the database."
+echo

bin/delete_member.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Delete any reference to an account of a given member by member_id
+# If second argument ('data_only') is set to non-zero,
+#  only delete  CS, PA, SA and LOGGING data but not MA data
+# If second argument ('data_only') is zero, delete MA data as well
+#
+# Usage: delete_member.sh member_id [data_only=1]
+
+if [ $# -lt 1 ]; then
+    echo "Usage: delete_member.sh member_id [data_only=1]"
+    exit
+else
+    MEMBER_ID=$1
+    DATA_ONLY=1;
+    if [ $# -gt 1 ]; then
+	DATA_ONLY=$2;
+    fi
+    if [ $DATA_ONLY -eq 0 ]; then
+	echo "delete from ma_member_attribute where member_id = '$MEMBER_ID'" | psql -U portal -h localhost portal
+	echo "delete from ma_member_privilege where member_id = '$MEMBER_ID'" | psql -U portal -h localhost portal
+	echo "delete from ma_inside_key where member_id = '$MEMBER_ID'" | psql -U portal -h localhost portal
+	echo "delete from ma_ssh_key where member_id = '$MEMBER_ID'" | psql -U portal -h localhost portal
+	echo "delete from ma_member where member_id = '$MEMBER_ID'" | psql -U portal -h localhost portal
+    fi
+    echo "delete from pa_project_member where member_id = '$MEMBER_ID'" | psql -U portal -h localhost portal
+    echo "delete from sa_slice_member where member_id = '$MEMBER_ID'" | psql -U portal -h localhost portal
+    echo "delete from logging_entry_attribute where event_id in (select id from logging_entry where user_id = '$MEMBER_ID')" | psql -U portal -h localhost portal
+    echo "delete from logging_entry where user_id = '$MEMBER_ID'" | psql -U portal -h localhost portal
+fi

bin/delete_project.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Delete any reference to a project of given project_id 
+# in LOGGING, CS, SA, PA data
+#
+# Usage: delete_project.sh project_id 
+
+if [ $# -lt 1 ]; then
+    echo "Usage: delete_project.sh project_id"
+    exit
+else
+    PROJECT_ID=$1
+    echo "delete from sa_slice_member where slice_id in (select slice_id from sa_slice where project_id = '$PROJECT_ID')" | psql -U portal -h localhost portal
+    echo "delete from sa_slice where project_id = '$PROJECT_ID'" | psql -U portal -h localhost portal
+    echo "delete from pa_project_member where project_id='$PROJECT_ID'" | psql -U portal -h localhost portal
+    echo "delete from pa_project where project_id = '$PROJECT_ID'" | psql -U portal -h localhost portal
+    echo "delete from logging_entry where id in (select event_id from logging_entry_attribute where attribute_name = 'PROJECT' and attribute_value = '$PROJECT_ID')" | psql -U portal -h localhost portal
+    echo "delete from logging_entry_attribute where attribute_name = 'PROJECT' and attribute_value = '$PROJECT_ID'" | psql -U portal -h localhost portal
+fi

bin/delete_slice.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# Delete any reference to a slice of given slice_id 
+# in LOGGING, CS, SA data
+#
+# Usage: delete_slice.sh slice_id 
+
+if [ $# -lt 1 ]; then
+    echo "Usage: delete_slice.sh slice_id"
+    exit
+else
+    SLICE_ID=$1
+    echo "delete from sa_slice_member where slice_id='$SLICE_ID'" | psql -U portal -h localhost portal
+    echo "delete from sa_slice where slice_id = '$SLICE_ID'" | psql -U portal -h localhost portal
+    echo "delete from logging_entry where id in (select event_id from logging_entry_attribute where attribute_name = 'SLICE' and attribute_value = '$SLICE_ID')" | psql -U portal -h localhost portal
+    echo "delete from logging_entry_attribute where attribute_name = 'SLICE' and attribute_value = '$SLICE_ID'" | psql -U portal -h localhost portal
+fi
+
+
+
+