Omnibus PR for our consideration 2019.10.04

.gitattributes

@@ -20,3 +20,7 @@
 *.PDF	 diff=astextplain
 *.rtf	 diff=astextplain
 *.RTF	 diff=astextplain
+
+sysadmin/files/referral-spam.conf diff=ansible-vault merge=binary
+sysadmin/templates/nginx_conf.j2 diff=ansible-vault merge=binary
+sysadmin/files/ssl-certs/opencontext_predb/* diff=ansible-vault merge=binary

Pipfile

@@ -0,0 +1,58 @@
+[[source]]
+name = "pypi"
+verify_ssl = true
+url = "https://pypi.org/simple"
+
+[packages]
+anyjson = ">=0.3.3"
+coverage = ">=4.0.3"
+coveralls = ">=1.1"
+Django = ">=2.0.7"
+django-debug-toolbar = ">=1.9"
+django-redis-cache = ">=1.7.0"
+django-reversion = ">=2.0.6"
+django-user_agents = ">=0.3.0"
+docopt = ">=0.6.2"
+feedparser = ">=5.2.1"
+geojson = ">=1.3"
+isodate = ">=0.5.4"
+jsonfield = ">=1.0.3"
+lxml = ">=3.8"
+mysolr = ">=0.8.3"
+numpy = ">=1.11.0"
+pandas = ">=0.24.0"
+"psycopg2" = ">=2.7.5"
+pyparsing = ">=2.1.1"
+python-dateutil = ">=2.5.3"
+pytz = ">=2016.4"
+rdflib = ">=4.2.1"
+rdflib-jsonld = ">=0.4.0"
+redis = ">=2.10.5"
+requests = ">=2.20.0"
+roman = ">=2.0.0"
+scipy = ">=0.17.0"
+six = ">=1.10.0"
+sqlparse = ">=0.2.4"
+titlecase = ">=0.9.0"
+ua-parser = ">=0.7.1"
+user-agents = ">=1.0.1"
+python-memcached = ">=1.59"
+pyproj = ">=1.9.5"
+atomicwrites = ">=1.2.1"
+attrs = ">=18.2.0"
+more-itertools = ">=4.3.0"
+"pathlib2" = ">=2.3.2"
+pluggy = ">=0.8.0"
+py = ">=1.7.0"
+pytest = ">=3.9.3"
+pytest-django = ">=3.4.3"
+Pillow = ">=5.2.0"
+PyYAML = ">=3.11"
+Shapely = ">=1.5.15"
+Unidecode = ">=0.4.19"
+uWSGI = ">=2.0.17.1"
+
+[dev-packages]
+
+[requires]
+python_version = "3.7"

conftest.py

@@ -0,0 +1,22 @@
+import pytest
+import json
+from opencontext_py import settings
+
+@pytest.fixture(autouse=True)
+def enable_db_access_for_all_tests(db):
+    pass
+
+@pytest.fixture(scope='session')
+def django_db_setup():
+
+    with open('secrets.json', 'r') as secrets_file:
+        secrets = json.loads(secrets_file.read())
+
+    settings.DATABASES['default'] = {
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': secrets.get('DATABASES_NAME'),
+        'USER': secrets.get('DATABASES_USER'),
+        'PASSWORD': secrets.get('DATABASES_PASSWORD'),
+        'HOST': secrets.get('DATABASES_HOST'),
+        'CONN_MAX_AGE': 600,
+    }

opencontext_py/apps/about/tests.py

@@ -7,10 +7,10 @@ class TestPredicateValues(TestCase):
 
     def setUp(self):
         self.oc_item = OCitem(
-            ).get_item('9E474B89-E36B-4B9D-2D38-7C7CCBDBB030')
+            ).get_item('FA6BFBFD-39EB-4474-A2D9-860B2D1B81A6')
         self.json_ld = self.oc_item.json_ld
         self.oc_label = self.json_ld['label']
 
     def test_get_oc_item_label(self):
         #oc_label = self.json_ld['label']
-        self.assertEqual(self.oc_label, 'Bone 4502')
+        self.assertEqual(self.oc_label, 'CAT # 20')

opencontext_py/apps/archive/tests.py

@@ -6,6 +6,7 @@
 from django.db import transaction
 from django.conf import settings
 from django.test import TestCase
+from django.test.client import Client
 from rdflib.plugin import register, Parser
 from rdflib import Graph, URIRef, Literal
 from rdflib.graph import ConjunctiveGraph
@@ -28,10 +29,15 @@ def setUp(self):
                  'rdflib_jsonld.parser',
                  'JsonLDParser')
         # self.graph = Graph()
-        self.proj_context_uri = 'http://127.0.0.1:8000/contexts/projects/3.json'
+        # self.proj_context_uri = 'http://127.0.0.1:8000/contexts/projects/3FAAA477-5572-4B05-8DC1-CA264FE1FC10.json'
+        self.proj_context_uri = '/contexts/projects/3FAAA477-5572-4B05-8DC1-CA264FE1FC10.json'
         self.context_file = settings.STATIC_IMPORTS_ROOT + '3-context.json'
         self.data_file = settings.STATIC_IMPORTS_ROOT + 'dt-bone.json'
-        self.context_str = self.request_json_str(self.proj_context_uri)
+        #self.context_str = self.request_json_str(self.proj_context_uri)
+        client = Client()
+        response = client.get(self.proj_context_uri, follow=True)
+        assert response.status_code in [200,301]
+        self.context_str = response.content
         self.data_str = self.load_json_file_str(self.data_file)
         g_context = ConjunctiveGraph(identifier=self.proj_context_uri)
         g_context.parse(data=self.context_str, format='json-ld')

opencontext_py/apps/contexts/tests.py

@@ -28,7 +28,8 @@ def setUp(self):
                  'rdflib_jsonld.parser',
                  'JsonLDParser')
         # self.graph = Graph()
-        self.proj_context_uri = 'http://127.0.0.1:8000/contexts/projects/3.json'
+        #self.proj_context_uri = 'http://127.0.0.1:8000/contexts/projects/3.json'
+        self.proj_context_uri = '/127.0.0.1:8000/contexts/projects/3.json'
         self.context_file = settings.STATIC_IMPORTS_ROOT + '3-context.json'
         self.data_file = settings.STATIC_IMPORTS_ROOT + 'dt-bone.json'
         self.context_str = self.request_json_str(self.proj_context_uri)

opencontext_py/apps/edit/inputs/fieldgroups/tests.py

@@ -7,10 +7,10 @@ class TestPredicateValues(TestCase):
 
     def setUp(self):
         self.oc_item = OCitem(
-            ).get_item('9E474B89-E36B-4B9D-2D38-7C7CCBDBB030')
+            ).get_item('FA6BFBFD-39EB-4474-A2D9-860B2D1B81A6')
         self.json_ld = self.oc_item.json_ld
         self.oc_label = self.json_ld['label']
 
     def test_get_oc_item_label(self):
         #oc_label = self.json_ld['label']
-        self.assertEqual(self.oc_label, 'Bone 4502')
+        self.assertEqual(self.oc_label, 'CAT # 20')

opencontext_py/apps/edit/inputs/inputfields/tests.py

@@ -7,10 +7,10 @@ class TestPredicateValues(TestCase):
 
     def setUp(self):
         self.oc_item = OCitem(
-            ).get_item('9E474B89-E36B-4B9D-2D38-7C7CCBDBB030')
+            ).get_item('FA6BFBFD-39EB-4474-A2D9-860B2D1B81A6')
         self.json_ld = self.oc_item.json_ld
         self.oc_label = self.json_ld['label']
 
     def test_get_oc_item_label(self):
         #oc_label = self.json_ld['label']
-        self.assertEqual(self.oc_label, 'Bone 4502')
+        self.assertEqual(self.oc_label, 'CAT # 20')

opencontext_py/apps/edit/inputs/inputrelations/tests.py

@@ -7,10 +7,10 @@ class TestPredicateValues(TestCase):
 
     def setUp(self):
         self.oc_item = OCitem(
-            ).get_item('9E474B89-E36B-4B9D-2D38-7C7CCBDBB030')
+            ).get_item('FA6BFBFD-39EB-4474-A2D9-860B2D1B81A6')
         self.json_ld = self.oc_item.json_ld
         self.oc_label = self.json_ld['label']
 
     def test_get_oc_item_label(self):
         #oc_label = self.json_ld['label']
-        self.assertEqual(self.oc_label, 'Bone 4502')
+        self.assertEqual(self.oc_label, 'CAT # 20')

opencontext_py/apps/edit/inputs/profiles/tests.py

@@ -7,10 +7,10 @@ class TestPredicateValues(TestCase):
 
     def setUp(self):
         self.oc_item = OCitem(
-            ).get_item('9E474B89-E36B-4B9D-2D38-7C7CCBDBB030')
+            ).get_item('FA6BFBFD-39EB-4474-A2D9-860B2D1B81A6')
         self.json_ld = self.oc_item.json_ld
         self.oc_label = self.json_ld['label']
 
     def test_get_oc_item_label(self):
         #oc_label = self.json_ld['label']
-        self.assertEqual(self.oc_label, 'Bone 4502')
+        self.assertEqual(self.oc_label, 'CAT # 20')

opencontext_py/apps/edit/inputs/rules/tests.py

@@ -7,10 +7,10 @@ class TestPredicateValues(TestCase):
 
     def setUp(self):
         self.oc_item = OCitem(
-            ).get_item('9E474B89-E36B-4B9D-2D38-7C7CCBDBB030')
+            ).get_item('FA6BFBFD-39EB-4474-A2D9-860B2D1B81A6')
         self.json_ld = self.oc_item.json_ld
         self.oc_label = self.json_ld['label']
 
     def test_get_oc_item_label(self):
         #oc_label = self.json_ld['label']
-        self.assertEqual(self.oc_label, 'Bone 4502')
+        self.assertEqual(self.oc_label, 'CAT # 20')

opencontext_py/apps/index/tests.py

@@ -7,10 +7,10 @@ class TestPredicateValues(TestCase):
 
     def setUp(self):
         self.oc_item = OCitem(
-            ).get_item('9E474B89-E36B-4B9D-2D38-7C7CCBDBB030')
+            ).get_item('FA6BFBFD-39EB-4474-A2D9-860B2D1B81A6')
         self.json_ld = self.oc_item.json_ld
         self.oc_label = self.json_ld['label']
 
     def test_get_oc_item_label(self):
         #oc_label = self.json_ld['label']
-        self.assertEqual(self.oc_label, 'Bone 4502')
+        self.assertEqual(self.oc_label, 'CAT # 20')

opencontext_py/apps/indexer/crawler.py

@@ -31,7 +31,7 @@ def __init__(self):
         index_single_document method. Just provide the document's UUID.
         For example:
 
-        crawler.index_single_document('9E474B89-E36B-4B9D-2D38-7C7CCBDBB030')
+        crawler.index_single_document('FA6BFBFD-39EB-4474-A2D9-860B2D1B81A6')
         '''
         # The list of Open Context items to crawl
         self.uuidlist = UUIDList().uuids
@@ -203,7 +203,7 @@ def index_single_document(self, uuid):
         UUID as an argument. For example:
 
         crawler = Crawler()
-        crawler.index_single_document('9E474B89-E36B-4B9D-2D38-7C7CCBDBB030')
+        crawler.index_single_document('FA6BFBFD-39EB-4474-A2D9-860B2D1B81A6')
         '''
         print('\nAttempting to index document ' + uuid + '...\n')
         start_time = time.time()

opencontext_py/apps/indexer/tests.py

@@ -7,10 +7,10 @@ class TestPredicateValues(TestCase):
 
     def setUp(self):
         self.oc_item = OCitem(
-            ).get_item('9E474B89-E36B-4B9D-2D38-7C7CCBDBB030')
+            ).get_item('FA6BFBFD-39EB-4474-A2D9-860B2D1B81A6')
         self.json_ld = self.oc_item.json_ld
         self.oc_label = self.json_ld['label']
 
     def test_get_oc_item_label(self):
         #oc_label = self.json_ld['label']
-        self.assertEqual(self.oc_label, 'Bone 4502')
+        self.assertEqual(self.oc_label, 'CAT # 20')

opencontext_py/apps/oai/tests.py

@@ -7,10 +7,10 @@ class TestPredicateValues(TestCase):
 
     def setUp(self):
         self.oc_item = OCitem(
-            ).get_item('9E474B89-E36B-4B9D-2D38-7C7CCBDBB030')
+            ).get_item('FA6BFBFD-39EB-4474-A2D9-860B2D1B81A6')
         self.json_ld = self.oc_item.json_ld
         self.oc_label = self.json_ld['label']
 
     def test_get_oc_item_label(self):
         #oc_label = self.json_ld['label']
-        self.assertEqual(self.oc_label, 'Bone 4502')
+        self.assertEqual(self.oc_label, 'CAT # 20')

opencontext_py/apps/ocitems/projects/permissions.py

@@ -56,7 +56,7 @@ def view_allowed(self, request):
     def edit_allowed(self, request):
         """ Checks to see if a user is allowed to edit a project """
         output = False
-        if request.user.is_authenticated():
+        if request.user.is_authenticated:
             if request.user.is_superuser:
                 # super users are super!
                 output = True

opencontext_py/apps/utilities/tests.py

@@ -7,10 +7,10 @@ class TestPredicateValues(TestCase):
 
     def setUp(self):
         self.oc_item = OCitem(
-            ).get_item('9E474B89-E36B-4B9D-2D38-7C7CCBDBB030')
+            ).get_item('FA6BFBFD-39EB-4474-A2D9-860B2D1B81A6')
         self.json_ld = self.oc_item.json_ld
         self.oc_label = self.json_ld['label']
 
     def test_get_oc_item_label(self):
         #oc_label = self.json_ld['label']
-        self.assertEqual(self.oc_label, 'Bone 4502')
+        self.assertEqual(self.oc_label, 'CAT # 20')

opencontext_py/libs/requestnegotiation.py

@@ -18,7 +18,7 @@ def anonymize_request(self, request):
         """ anonymizes a request by flushing session cookies for users
             that are not logged in
         """
-        if not request.user.is_authenticated():
+        if not request.user.is_authenticated:
             # the user is not authenticated, therefore
             # we will default to expire the session cookie for a user after the default
             pass

opencontext_py/settings.py

@@ -209,7 +209,7 @@ def get_secret(setting, secrets=secrets):
     'django_user_agents',
 )
 
-MIDDLEWARE_CLASSES = (
+MIDDLEWARE = (
     'debug_toolbar.middleware.DebugToolbarMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.common.CommonMiddleware',

opencontext_py/tests/README.md

@@ -0,0 +1,7 @@
+# About Tests
+
+The tests directory should be subdivided into 3 directories with different kinds of tests. This will be:
+
+1. `/unit`: These will test functions and not require use of the database.
+2. `/integration`: These are tests need a (temporary) database, but can work with fake data.
+3. `/regression`: These tests will either require or will be easiest to conduct with real data in the Open Context database.