Merge laest changes from develop

CHANGES

@@ -5,22 +5,34 @@
 
 = GENI Clearinghouse Release Notes =
 
+== 2.1 ==
+ * Migrate CH tables from geni-portal to geni-ch (#103).
+ * Support lists of project_ids in option for lookup_project_attributes (#391).
+ * Return most recent slice from SA.lookup and SA.lookup_slices (#393)
+ * Allow JSON booleans for boolean type arguments to API calls (#394)
+
+== 2.0 ==
+ * Add procedure to add new aggregate (#383).
+ * Minor tweaks to `portal_stats.sql`
+
 == 1.29 ==
- * `geni-check-errors` now suppresses certificate generation output, 
+ * remove hard-coded names in SAv1Implementation.py, 
+   MAv1Implementation.py and cert-utils.py (#25)
+ * Add geni-list-idp-members query script to print the number of
+   users per IDP (by eppn suffix). (#263).
+ * Add geni-list-pending-requests query script to print all pending
+   project join requests (project_name, requestor username and request
+   time) (#263)
+ * `geni-check-errors` now suppresses certificate generation output,
    Certificate Verification errors from apache when someone accesses ch.geni.net
    (like PG AMs), messages about users creating or renewing certs, messages
    about members setting their own attributes or their irods_username,
    and messages about failing to access the system during maintenance.
    It also looks at the older `chapi.log.1` and `error.log.1`. (#360)
  * `geni-check-errors` now also ignores collector tool speaksfor. (#361)
- * remove hard-coded names in SAv1Implementation.py, 
-   MAv1Implementation.py and cert-utils.py (#25)
- * Add geni-list-idp-members query script to print the number of
-   users per IDP (by eppn suffix). (#364).
- * Add geni-list-pending-requests query script to print all pending
-   project join requests (project_name, requestor username and request time)
  * Add iMinds w-iLab.t and Virtual Wall 1 aggregates (#367)
  * Migrate management scripts from geni-portal to geni-ch (#101)
+ * Add Kaiserslautern OpenGENI aggreate (#374)
 
 == 1.28 ==
  * Update aggregate info for some stitchable aggregates

Makefile.am

@@ -1,6 +1,6 @@
 ## Process this file with automake to produce Makefile.in
 
-SUBDIRS = plugins tools etc bin man data
+SUBDIRS = plugins tools etc bin man data db
 
 .PHONY: $(SUBDIRS)
 

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT([geni-chapi], [1.29], [[email protected]])
+AC_INIT([geni-chapi], [2.1], [[email protected]])
 AM_INIT_AUTOMAKE([foreign -Wall -Wno-portability])
 AC_PROG_MKDIR_P
 AC_PROG_INSTALL
@@ -20,5 +20,5 @@ AM_CONDITIONAL([GPO_LAB], [test x$gpo_lab = xtrue])
 AM_CONDITIONAL(INSTALL_GITHASH, [test -f etc/geni-chapi-githash])
 
 AC_CONFIG_FILES([Makefile plugins/Makefile tools/Makefile etc/Makefile])
-AC_CONFIG_FILES([bin/Makefile man/Makefile data/Makefile])
+AC_CONFIG_FILES([bin/Makefile man/Makefile data/Makefile db/Makefile])
 AC_OUTPUT

data/Makefile.am

@@ -27,6 +27,7 @@ AM_SQL = \
 	sr/sql/add-gpo-og.sql \
 	sr/sql/add-im-wilab.sql \
 	sr/sql/add-im-vw1.sql \
+	sr/sql/add-ukl-og.sql \
 	sr/sql/add-moxi-ig-of.sql \
 	sr/sql/add-moxi-ig.sql \
 	sr/sql/add-moxi-of.sql \
@@ -116,6 +117,7 @@ sr/sql/add-gpo-eg.sql: $(srcdir)/sr/sql/add-gpo-eg.sql.in
 sr/sql/add-gpo-og.sql: $(srcdir)/sr/sql/add-gpo-og.sql.in
 sr/sql/add-im-wilab.sql: $(srcdir)/sr/sql/add-im-wilab.sql.in
 sr/sql/add-im-vw1.sql: $(srcdir)/sr/sql/add-im-vw1.sql.in
+sr/sql/add-ukl-og.sql: $(srcdir)/sr/sql/add-ukl-og.sql.in
 sr/sql/add-moxi-ig-of.sql: $(srcdir)/sr/sql/add-moxi-ig-of.sql.in
 sr/sql/add-moxi-ig.sql: $(srcdir)/sr/sql/add-moxi-ig.sql.in
 sr/sql/add-moxi-of.sql: $(srcdir)/sr/sql/add-moxi-of.sql.in
@@ -195,6 +197,7 @@ dist_srcerts_DATA = \
 	sr/certs/im-wilab-ssl.pem \
 	sr/certs/im-vw1-cm.pem \
 	sr/certs/im-vw1-ssl.pem \
+	sr/certs/ukl-og.pem \
 	sr/certs/moxi-ig-boss.pem \
 	sr/certs/moxi-ig-cm.pem \
 	sr/certs/moxi-ig-of.pem \

data/sr/aggdata.csv

@@ -17,6 +17,7 @@ gpo-eg-of,https://bbn-hn.exogeni.net:3626/foam/gapi/2,gpo-eg-of.pem,GPO ExoGENI
 gpo-og,https://bbn-cam-ctrl-1.gpolab.bbn.com:5002,gpo-og.pem,GPO OpenGENI,GPO OpenGENI Rack,urn:publicid:IDN+bbn-cam-ctrl-1.gpolab.bbn.com+authority+am,gpo-og.pem,ui_other_am,ui_prod_cat ui_compute_cat,Y
 im-wilab,https://www.wilab2.ilabt.iminds.be:12369/protogeni/xmlrpc/am/2.0,im-wilab-cm.pem,iMinds w-iLab.t,iMinds w-iLab.t,urn:publicid:IDN+wilab2.ilabt.iminds.be+authority+cm,im-wilab-ssl.pem,ui_instageni_am,ui_federated_cat ui_compute_cat,Y
 im-vw1,https://www.wall1.ilabt.iminds.be:12369/protogeni/xmlrpc/am/2.0,im-vw1-cm.pem,iMinds Virtual Wall 1,iMinds Virtual Wall 1,urn:publicid:IDN+wall1.ilabt.iminds.be+authority+cm,im-vw1-ssl.pem,ui_instageni_am,ui_federated_cat ui_compute_cat,Y
+ukl-og,https://glab077.e4.ukl.german-lab.de:5002,ukl-og.pem,Kaiserslautern OpenGENI,Kaiserslautern OpenGENI Rack,urn:publicid:IDN+glab077.e4.ukl.german-lab.de:gcf+authority+am,ukl-og.pem,ui_other_am,ui_federated_cat ui_compute_cat,Y
 moxi-ig,https://instageni.iu.edu:12369/protogeni/xmlrpc/am/2.0,moxi-ig-cm.pem,MOXI InstaGENI,MOXI InstaGENI Rack,urn:publicid:IDN+instageni.iu.edu+authority+cm,moxi-ig-boss.pem,ui_instageni_am,ui_prod_cat ui_compute_cat,Y
 moxi-ig-of,https://foam.instageni.iu.edu:3626/foam/gapi/2,moxi-ig-of.pem,MOXI InstaGENI OpenFlow,MOXI InstaGENI Rack OpenFlow,urn:publicid:IDN+openflow:foam:foam.instageni.iu.edu+authority+am,moxi-ig-of.pem,ui_foam_am,ui_prod_cat ui_network_cat,N
 moxi-of,https://moxifoam.600wchicag.omnipop.cic.net:3626/foam/gapi/2,moxi-of.pem,MOXI OpenFlow,MOXI OpenFlow,urn:publicid:IDN+openflow:foam:moxifoam.ictc.indiana.gigapop.net+authority+am,moxi-of.pem,ui_other_am,ui_prod_cat ui_network_cat,N

data/sr/certs/ukl-og.pem

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIClDCCAf2gAwIBAgIBAzANBgkqhkiG9w0BAQQFADA/MT0wOwYDVQQDEzRnZW5p
+Ly9nbGFiMDc3LmU0LnVrbC5nZXJtYW4tbGFiLmRlLy9nY2YuYXV0aG9yaXR5LnNh
+MB4XDTE1MDEwNDIxNTEyMloXDTIwMDEwMzIxNTEyMlowRTFDMEEGA1UEAxM6Z2Vu
+aS8vZ2xhYjA3Ny5lNC51a2wuZ2VybWFuLWxhYi5kZS8vZ2NmLy9ncmFtLmF1dGhv
+cml0eS5hbTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAr4tDyO8Vu5AFH0SL
+px80sQm1HxqqB+6EOx8QLK532UiLyzPx2t0e3ToRtYkeYaG7CuAkc5qNWAeemd5I
+ypDURskexLctCCv7xpLl2HfNrpmZBGL8xOtYVeQ+de+vI/xCLMfIR36Z8QqPF51E
+V3WOpKfUUpf+VgBck8NjeASw0WkCAwEAAaOBmTCBljAPBgNVHRMBAf8EBTADAQH/
+MIGCBgNVHREEezB5hkh1cm46cHVibGljaWQ6SUROK2dlbmk6Z2xhYjA3Ny5lNC51
+a2wuZ2VybWFuLWxhYi5kZTpnY2Y6Z3JhbSthdXRob3JpdHkrYW2GLXVybjp1dWlk
+OjdiNWZmMmRmLTYxMTgtNDJjNS1iYTllLWNmNWQwMDllMTI0YTANBgkqhkiG9w0B
+AQQFAAOBgQCFN3GfxEpPhf7T3XZFwDqXvBCGduPevGpYwxinG/Qp1Q60qiO8Viit
+WBCoJTZWK5ZcBj5tMEQ77JBXNxXi6z22b92cIBlOgdaUJpN2mzODDIdYSfQTQT6q
+EMKHzTKHJ5juQDLee4UBdjgRKLpxnZQHM8ZWJKc9nAGHjOuyyxaixA==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIICiDCCAfGgAwIBAgIBAzANBgkqhkiG9w0BAQQFADA/MT0wOwYDVQQDEzRnZW5p
+Ly9nbGFiMDc3LmU0LnVrbC5nZXJtYW4tbGFiLmRlLy9nY2YuYXV0aG9yaXR5LnNh
+MB4XDTE1MDEwNDIxNTEyMloXDTIwMDEwMzIxNTEyMlowPzE9MDsGA1UEAxM0Z2Vu
+aS8vZ2xhYjA3Ny5lNC51a2wuZ2VybWFuLWxhYi5kZS8vZ2NmLmF1dGhvcml0eS5z
+YTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAv76cqQLqz1RBVvORMNY0KURt
+4KOudBeQDyAQWpEHosYaFY+QR2uSLjxrILdtEekixkMAQmgBE0jFRCDFT7RPfDvV
+cu8ZSsurnPJtDjQmKh+L5C74kk1iJPXk1zgmZ15JZi65rJwdWUqkNFj7IVIvcJ3o
+/Hj8WoAaIg2zJ40WVSUCAwEAAaOBkzCBkDAPBgNVHRMBAf8EBTADAQH/MH0GA1Ud
+EQR2MHSGQ3VybjpwdWJsaWNpZDpJRE4rZ2VuaTpnbGFiMDc3LmU0LnVrbC5nZXJt
+YW4tbGFiLmRlOmdjZithdXRob3JpdHkrc2GGLXVybjp1dWlkOjk4NTMwMzhlLTJh
+ODUtNGZhZi05ZTkyLTljYzA0ZGJhMzIzNjANBgkqhkiG9w0BAQQFAAOBgQBCy8AO
+Nfyyrf5j+pPbjxLhjuoHNkNtlsZDtbMEsa/knoNSYjNChUdQp9beCMfC1MZub8mC
+htsjJlYqn++QBpwZTyb3gIEld0PRsT4ibbtkjcgrDKqYulChGmoJ6AO8U8Ijyul+
+foxjB/BExNJkNRxUsnFygV+RtFuPFTPz4Eb+Iw==
+-----END CERTIFICATE-----

db/Makefile.am

@@ -0,0 +1,50 @@
+# Put the db files in a subdirectory of pkgdatadir
+dbdir = $(pkgdatadir)/db
+
+nobase_dist_db_DATA = \
+	cs/postgresql/data.sql \
+	cs/postgresql/disable_lockdown.sql \
+	cs/postgresql/enable_lockdown.sql \
+	cs/postgresql/schema.sql \
+	cs/postgresql/update-1.sql \
+	cs/postgresql/update-2.sql \
+	cs/postgresql/update-3.sql \
+	cs/postgresql/update-4.sql \
+	cs/postgresql/update-5.sql \
+	cs/postgresql/update-6.sql \
+	cs/postgresql/update-7.sql \
+	cs/postgresql/update-8.sql \
+	logging/postgresql/data.sql \
+	logging/postgresql/schema.sql \
+	logging/postgresql/update-1.sql \
+	logging/postgresql/update-2.sql \
+	logging/postgresql/update-3.sql \
+	ma/postgresql/data.sql \
+	ma/postgresql/schema.sql \
+	ma/postgresql/update-1.sql \
+	ma/postgresql/update-2.sql \
+	ma/postgresql/update-3.sql \
+	ma/postgresql/update-4.sql \
+	ma/postgresql/update-5.sql \
+	migration/migrate-assertions.sql \
+	migration/sliver-info.sql \
+	pa/postgresql/data.sql \
+	pa/postgresql/schema.sql \
+	pa/postgresql/update-1.sql \
+	pa/postgresql/update-2.sql \
+	pa/postgresql/update-3.sql \
+	pa/postgresql/update-4.sql \
+	pa/postgresql/update-5.sql \
+	sa/postgresql/data.sql \
+	sa/postgresql/README.txt \
+	sa/postgresql/schema.sql \
+	sa/postgresql/update-1.sql \
+	sa/postgresql/update-2.sql \
+	sa/postgresql/update-3.sql \
+	sr/postgresql/data.sql \
+	sr/postgresql/README.txt \
+	sr/postgresql/schema.sql \
+	sr/postgresql/update-1.sql \
+	sr/postgresql/update-2.sql \
+	sr/postgresql/update-3.sql \
+	sr/postgresql/update-4.sql

db/cs/postgresql/data.sql

@@ -0,0 +1,73 @@
+
+-- ----------------------------------------------------------------------
+-- A few initial records to insert into the database
+-- ----------------------------------------------------------------------
+
+-- Define attributes
+INSERT INTO cs_attribute (id, name) values (1, 'LEAD');
+INSERT INTO cs_attribute (id, name) values (2, 'ADMIN');
+INSERT INTO cs_attribute (id, name) values (3, 'MEMBER');
+INSERT INTO cs_attribute (id, name) values (4, 'AUDITOR');
+INSERT INTO cs_attribute (id, name) values (5, 'OPERATOR');
+
+-- Define  privileges
+INSERT INTO cs_privilege (id, name) values (1, 'DELEGATE'); 
+INSERT INTO cs_privilege (id, name) values (2, 'READ');
+INSERT INTO cs_privilege (id, name) values (3, 'WRITE');
+INSERT INTO cs_privilege (id, name) values (4, 'USE');
+
+-- Define context types
+insert into cs_context_type (id, name) values (1, 'PROJECT');
+insert into cs_context_type (id, name) values (2, 'SLICE');
+insert into cs_context_type (id, name) values (3, 'RESOURCE');
+insert into cs_context_type (id, name) values (4, 'SERVICE');
+insert into cs_context_type (id, name) values (5, 'MEMBER');
+
+-- Define actions
+insert into cs_action (name, privilege, context_type) values ('project_read', 2, 1);
+insert into cs_action (name, privilege, context_type) values ('project_write', 3, 1);
+insert into cs_action (name, privilege, context_type) values ('project_use', 4, 1);
+insert into cs_action (name, privilege, context_type) values ('slice_read', 2, 2);
+insert into cs_action (name, privilege, context_type) values ('slice_write', 3, 2);
+insert into cs_action (name, privilege, context_type) values ('slice_use', 4, 2);
+insert into cs_action (name, privilege, context_type) values ('create_project', 3, 3);
+insert into cs_action (name, privilege, context_type) values ('administer_members', 3, 5);
+
+-- Define initial set of policies based on PROJECT/SLICE READ/WRITE/USE
+insert into cs_policy (attribute, context_type, privilege) values ('1', '1','2');
+insert into cs_policy (attribute, context_type, privilege) values ('1', '1','3');
+insert into cs_policy (attribute, context_type, privilege) values ('1', '1','4');
+insert into cs_policy (attribute, context_type, privilege) values ('2', '1','2');
+insert into cs_policy (attribute, context_type, privilege) values ('2', '1','3');
+insert into cs_policy (attribute, context_type, privilege) values ('2', '1','4');
+insert into cs_policy (attribute, context_type, privilege) values ('3', '1','2');
+insert into cs_policy (attribute, context_type, privilege) values ('3', '1','4');
+insert into cs_policy (attribute, context_type, privilege) values ('4', '1','2');
+insert into cs_policy (attribute, context_type, privilege) values ('5', '1','2');
+insert into cs_policy (attribute, context_type, privilege) values ('5', '1','3');
+insert into cs_policy (attribute, context_type, privilege) values ('5', '1','4');
+insert into cs_policy (attribute, context_type, privilege) values ('1', '2','2');
+insert into cs_policy (attribute, context_type, privilege) values ('1', '2','3');
+insert into cs_policy (attribute, context_type, privilege) values ('1', '2','4');
+insert into cs_policy (attribute, context_type, privilege) values ('2', '2','2');
+insert into cs_policy (attribute, context_type, privilege) values ('2', '2','3');
+insert into cs_policy (attribute, context_type, privilege) values ('2', '2','4');
+insert into cs_policy (attribute, context_type, privilege) values ('3', '2','2');
+insert into cs_policy (attribute, context_type, privilege) values ('3', '2','4');
+insert into cs_policy (attribute, context_type, privilege) values ('4', '2','2');
+insert into cs_policy (attribute, context_type, privilege) values ('5', '2','2');
+insert into cs_policy (attribute, context_type, privilege) values ('5', '2','3');
+insert into cs_policy (attribute, context_type, privilege) values ('5', '2','4');
+insert into cs_policy (attribute, context_type, privilege) values ('1', '3','3');
+insert into cs_policy (attribute, context_type, privilege) values ('5', '3','1');
+insert into cs_policy (attribute, context_type, privilege) values ('5', '3','2');
+insert into cs_policy (attribute, context_type, privilege) values ('5', '3','3');
+insert into cs_policy (attribute, context_type, privilege) values ('5', '4','1');
+insert into cs_policy (attribute, context_type, privilege) values ('5', '4','2');
+insert into cs_policy (attribute, context_type, privilege) values ('5', '4','3');
+insert into cs_policy (attribute, context_type, privilege) values ('5', '5','1');
+insert into cs_policy (attribute, context_type, privilege) values ('5', '5','2');
+insert into cs_policy (attribute, context_type, privilege) values ('5', '5','3');
+insert into cs_policy (attribute, context_type, privilege) values ('5', '3','4');
+insert into cs_policy (attribute, context_type, privilege) values ('5', '4','4');
+insert into cs_policy (attribute, context_type, privilege) values ('5', '5','4');

db/cs/postgresql/disable_lockdown.sql

@@ -0,0 +1 @@
+update cs_action set privilege = privilege + 100 where privilege < 0;

db/cs/postgresql/enable_lockdown.sql

@@ -0,0 +1,50 @@
+update cs_action
+set privilege = privilege - 100 
+where 
+privilege > 0 and
+name in ('create_assertion', 
+      'create_policy'
+      'create_assertion',
+      'create_policy',
+      'renew_assertion',
+      'delete_policy',
+--      'query_assertions',
+--      'query_policies',
+--      'lookup_slice',
+--      'lookup_slices',
+--      'lookup_slice_ids',
+--      'get_slice_credential',
+--      'add_slivers',
+--      'delete_slivers',
+      'renew_slice',
+--      'get_slice_members',
+--      'get_slices_for_member',
+--      'lookup_slices_by_ids',
+--      'get_slice_members_for_project',
+--      'list_resources',
+--      'get_services',
+--      'get_services_of_type',
+      'register_service',
+      'remove_service',
+      'create_project',
+      'delete_project',
+--      'get_projects',
+--      'get_project_by_lead',
+--      'lookup_project',
+--      'update_project',
+--      'get_project_members',
+--      'get_projects_for_member',
+      'administer_resources',
+      'administer_services',
+      'administer_members',
+      'change_lead',
+      'add_project_member',
+      'remove_project_member',
+      'change_member_role',
+      'remove_slice_member',
+      'add_slice_member',
+      'change_slice_member_role',
+      'create_slice',
+      'invite_member',
+      'modify_slice_membership',
+      'modify_project_membership');

db/cs/postgresql/schema.sql

@@ -0,0 +1,72 @@
+-- Tables for CS (Credential Store) of GENI Prototype Clearinghouse
+
+-- avoid innocuous NOTICEs about automatic sequence creation
+set client_min_messages='WARNING';
+
+-- Tell psql to stop on an error. Default behavior is to proceed.
+\set ON_ERROR_STOP 1
+
+-- ----------------------------------------------------------------------
+--
+-- ----------------------------------------------------------------------
+-- Drop the data first, then the type.
+DROP TABLE IF EXISTS cs_assertion CASCADE;
+DROP TABLE IF EXISTS cs_policy CASCADE;
+DROP TABLE IF EXISTS cs_action CASCADE;
+DROP TABLE IF EXISTS cs_attribute CASCADE;
+DROP TABLE IF EXISTS cs_privilege CASCADE;
+DROP TABLE IF EXISTS cs_context_type CASCADE;
+
+-- List of all known attributes/roles on a principal
+CREATE TABLE cs_attribute (
+   id SERIAL PRIMARY KEY,
+   name VARCHAR NOT NULL UNIQUE
+);
+
+-- List of all known privileges that a principal may take
+CREATE TABLE cs_privilege  (
+   id SERIAL PRIMARY KEY,
+   name VARCHAR NOT NULL UNIQUE
+);
+
+-- A mapping of context type ID to name
+CREATE TABLE cs_context_type (
+  id SERIAL PRIMARY KEY,
+  name VARCHAR NOT NULL UNIQUE
+);
+
+-- List of all known actions and the required privilege and context type
+CREATE TABLE cs_action (
+   id SERIAL PRIMARY KEY,
+   name VARCHAR NOT NULL,
+   privilege int,
+   context_type int NOT NULL REFERENCES cs_context_type(id)
+);
+
+-- An assertion is a signed statement that a given principal has a given 
+-- attribute, possibly in a given context
+CREATE TABLE cs_assertion (
+  id SERIAL,
+  signer UUID,
+  principal UUID NOT NULL,
+  attribute INT NOT NULL REFERENCES cs_attribute(id), -- Index into cs_attribute table
+  context_type INT NOT NULL REFERENCES cs_context_type(id), -- 0 = NONE, 1 = PROJECT, 2 = SLICE, 3 = SLIVER
+  context UUID, 
+  expiration TIMESTAMP,
+  assertion_cert VARCHAR,
+  PRIMARY KEY (id)
+);
+-- can signer, principal, context by authorities who aren't in tables?
+
+-- A policy is a signed statement that a given holder of a given attribute
+-- has a given privilege. Again, this is potentially context dependent.
+CREATE TABLE cs_policy (
+  id SERIAL PRIMARY KEY,
+  signer UUID,
+  attribute INT NOT NULL REFERENCES cs_attribute(id), -- Index into cs_attribute
+  context_type INT NOT NULL REFERENCES cs_context_type(id), -- 0 = NONE, 1 = PROJECT, 2 = SLICE, 3 = SLIVER
+  privilege INT NOT NULL REFERENCES cs_privilege(id), -- Index into cs_privilege
+  policy_cert VARCHAR
+);
+
+