Merge branch 'station_restriction_wildcards' into geothermie2

.travis.yml

@@ -1,20 +1,25 @@
 language: python
 
+dist: trusty
+
+sudo: false
+
 python:
-  - "3.4"
   - "3.5"
+  - "3.6"
 
 env:
   global:
     secure: uH0xfi+u/FkddzkNdffIFMEv4edgHIE35no4E1nTZPCu8P3yraN9LOnfq+METCiHbFlC2O/6x32Azpg7x+jeOzfxdwCh7t5IpMg83cFV0wLZWqt940WV02LZTESjmGn5qFcJIp31pRZGxsr0S//TkfUrVNviwQ9j5kgztp7aCPfAiYYGrFEPaHuW8BiwZLVO6orAx3l/TPN3QeuF0nG0PVQiHmxbYhNTXVbjlYXRFW0swAfPBwit7yjPInqIpFbDD4slC5mhNS/EBffRE7kccPxLQAeyM69QVTkS0lHkgpEcir7NTLfKphA1uHoLtx+WutK8KwIQfzdhWOYzm0W8uVDUhAkL/p+bXT1SWMYGAtQTf6swatVjZgb0Hqb6xMXmhzMSb7vl/zhFTxkG+9TLF6edanj33j5I6wtxBUmB359Xaq9reNK1Yk6SuCAD56Yg90wHeEhgrxRURMWvjy1rV4vFPWRUx84wNdgHaTf89hiT9Vdp74eD8xqWEMgyOmgUfczJTqFcoNzfE1NKw9sycJkbuV4Sb/asjYGGadN+XTJBOAimjKj1xy3bbM2Aso6TuEMN15DE1Z9i3Ct9H9Lg9ThZvlE5EXJ3v+gP9XX81MpPyZ9pf84X1tNIiYwTzkg6JBwYiUVrf3P6M9pIKwvHSrSMos3Tn8tlIZHy4RTir/E=
 
-services:
-    - postgresql
-
 addons:
-    postgresql: "9.4"
+    postgresql: "9.6"
+    apt:
+        packages:
+            - postgresql-9.6-postgis-2.3
 
 before_script:
+    - psql -U postgres -c "create extension postgis"
     - createuser --superuser test_jane
 
 before_install:
@@ -26,10 +31,12 @@ before_install:
   - conda create --yes -n condaenv python=$TRAVIS_PYTHON_VERSION
   - conda install --yes -n condaenv pip
   - source activate condaenv
-  - conda install --yes -c obspy obspy psycopg2 markdown flake8 gdal pyyaml pip
+  - conda install --yes -c conda-forge obspy psycopg2 markdown flake8 gdal pyyaml pip
   - pip install codecov "django>=1.9,<1.10" djangorestframework djangorestframework-gis djangorestframework-jsonp djangorestframework-xml djangorestframework-yaml django-cors-headers django-debug-toolbar django-plugins defusedxml geojson markdown mkdocs mkdocs-bootswatch
   # Copy local_settings template.
   - cp $TRAVIS_BUILD_DIR/src/jane/local_settings.py.example $TRAVIS_BUILD_DIR/src/jane/local_settings.py
+  # Make sure django picks the correct geos library.
+  - echo -e "\n\nGDAL_LIBRARY_PATH = '/home/travis/miniconda/envs/condaenv/lib/libgdal.so'" >> $TRAVIS_BUILD_DIR/src/jane/local_settings.py
 
 install:
   - git version

docs/docs/waveforms.md

@@ -145,6 +145,11 @@ per-station granularity. To do that, add a new restriction in the admin
 interface. As soon as a restriction has been added it will be considered 
 protected and only users that are part of the restriction will still be able
 to access them.
+Restrictions can also be defined with a single asterisk (`*`) in the station
+(or network) code field, to make the restriction apply to all stations across a
+specific network (to apply to all networks across a specific station code). Use
+a single asterisk in *both* network and station code fields to add a
+restriction on *all* stations.
 
 ![Add waveform restriction](./images/add_waveform_restriction.png)
 

src/jane/documents/models.py

@@ -175,8 +175,9 @@ def add_or_modify_document(self, document_type, name, data, user):
         # Calculate the hash upfront to not upload any duplicates.
         sha1 = hashlib.sha1(data).hexdigest()
         if Document.objects.filter(sha1=sha1).exists():
-            raise JaneDocumentAlreadyExists("Data already exists in the "
-                                            "database.")
+            raise JaneDocumentAlreadyExists(
+                "Data already exists in the database and document is "
+                "identical according to its hash.")
 
         try:
             document = Document.objects.get(

src/jane/fdsnws/tests/test_station_1.py

@@ -423,6 +423,138 @@ def test_restrictions(self):
                                    **auth_headers)
         self.assertEqual(response.status_code, 204)
 
+    def test_restrictions_asterisk_network_and_station(self):
+        """
+        Tests if the waveform restrictions actually work as expected.
+        """
+        # No restrictions currently apply - we should get something.
+        response = self.client.get('/fdsnws/station/1/query')
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue('OK' in response.reason_phrase)
+        inv = obspy.read_inventory(io.BytesIO(response.getvalue()))
+        self.assertEqual(inv.get_contents()["stations"],
+                         ["BW.ALTM (Beilngries, Bavaria, BW-Net)"])
+
+        # add restriction on all stations
+        r = Restriction.objects.get_or_create(network="*", station="*")[0]
+        r.users.add(User.objects.filter(username='random')[0])
+        r.save()
+
+        # Now the same query should no longer return something as the
+        # station has been restricted.
+        response = self.client.get('/fdsnws/station/1/query')
+        self.assertEqual(response.status_code, 204)
+
+        # The correct user can still get the stations.
+        response = self.client.get('/fdsnws/station/1/queryauth',
+                                   **self.valid_auth_headers)
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue('OK' in response.reason_phrase)
+        inv = obspy.read_inventory(io.BytesIO(response.getvalue()))
+        self.assertEqual(inv.get_contents()["stations"],
+                         ["BW.ALTM (Beilngries, Bavaria, BW-Net)"])
+
+        # Make another user that has not been added to this restriction - he
+        # should not be able to retrieve it.
+        self.client.logout()
+        User.objects.get_or_create(
+            username='some_dude', password=make_password('some_dude'))[0]
+        credentials = base64.b64encode(b'some_dude:some_dude')
+        auth_headers = {
+            'HTTP_AUTHORIZATION': 'Basic ' + credentials.decode("ISO-8859-1")
+        }
+        response = self.client.get('/fdsnws/station/1/queryauth',
+                                   **auth_headers)
+        self.assertEqual(response.status_code, 204)
+
+    def test_restrictions_asterisk_network(self):
+        """
+        Tests if the waveform restrictions actually work as expected.
+        """
+        # No restrictions currently apply - we should get something.
+        response = self.client.get('/fdsnws/station/1/query')
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue('OK' in response.reason_phrase)
+        inv = obspy.read_inventory(io.BytesIO(response.getvalue()))
+        self.assertEqual(inv.get_contents()["stations"],
+                         ["BW.ALTM (Beilngries, Bavaria, BW-Net)"])
+
+        # add restriction on ALTM stations
+        r = Restriction.objects.get_or_create(network="*", station="ALTM")[0]
+        r.users.add(User.objects.filter(username='random')[0])
+        r.save()
+
+        # Now the same query should no longer return something as the
+        # station has been restricted.
+        response = self.client.get('/fdsnws/station/1/query')
+        self.assertEqual(response.status_code, 204)
+
+        # The correct user can still get the stations.
+        response = self.client.get('/fdsnws/station/1/queryauth',
+                                   **self.valid_auth_headers)
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue('OK' in response.reason_phrase)
+        inv = obspy.read_inventory(io.BytesIO(response.getvalue()))
+        self.assertEqual(inv.get_contents()["stations"],
+                         ["BW.ALTM (Beilngries, Bavaria, BW-Net)"])
+
+        # Make another user that has not been added to this restriction - he
+        # should not be able to retrieve it.
+        self.client.logout()
+        User.objects.get_or_create(
+            username='some_dude', password=make_password('some_dude'))[0]
+        credentials = base64.b64encode(b'some_dude:some_dude')
+        auth_headers = {
+            'HTTP_AUTHORIZATION': 'Basic ' + credentials.decode("ISO-8859-1")
+        }
+        response = self.client.get('/fdsnws/station/1/queryauth',
+                                   **auth_headers)
+        self.assertEqual(response.status_code, 204)
+
+    def test_restrictions_asterisk_station(self):
+        """
+        Tests if the waveform restrictions actually work as expected.
+        """
+        # No restrictions currently apply - we should get something.
+        response = self.client.get('/fdsnws/station/1/query')
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue('OK' in response.reason_phrase)
+        inv = obspy.read_inventory(io.BytesIO(response.getvalue()))
+        self.assertEqual(inv.get_contents()["stations"],
+                         ["BW.ALTM (Beilngries, Bavaria, BW-Net)"])
+
+        # add restriction on all BW-network stations
+        r = Restriction.objects.get_or_create(network="BW", station="*")[0]
+        r.users.add(User.objects.filter(username='random')[0])
+        r.save()
+
+        # Now the same query should no longer return something as the
+        # station has been restricted.
+        response = self.client.get('/fdsnws/station/1/query')
+        self.assertEqual(response.status_code, 204)
+
+        # The correct user can still get the stations.
+        response = self.client.get('/fdsnws/station/1/queryauth',
+                                   **self.valid_auth_headers)
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue('OK' in response.reason_phrase)
+        inv = obspy.read_inventory(io.BytesIO(response.getvalue()))
+        self.assertEqual(inv.get_contents()["stations"],
+                         ["BW.ALTM (Beilngries, Bavaria, BW-Net)"])
+
+        # Make another user that has not been added to this restriction - he
+        # should not be able to retrieve it.
+        self.client.logout()
+        User.objects.get_or_create(
+            username='some_dude', password=make_password('some_dude'))[0]
+        credentials = base64.b64encode(b'some_dude:some_dude')
+        auth_headers = {
+            'HTTP_AUTHORIZATION': 'Basic ' + credentials.decode("ISO-8859-1")
+        }
+        response = self.client.get('/fdsnws/station/1/queryauth',
+                                   **auth_headers)
+        self.assertEqual(response.status_code, 204)
+
 
 class Station1LiveServerTestCase(LiveServerTestCase):
     """

src/jane/settings.py

@@ -32,18 +32,12 @@
 DATE_FORMAT = "Y-m-d"
 
 
-if DEPLOYED:
-    MEDIA_ROOT = '/home/django/www/media'
-    MEDIA_URL = '/media/'
-    STATIC_ROOT = '/home/django/www/static'
-    STATIC_URL = '/static/'
-else:
-    MEDIA_ROOT = os.path.abspath(os.path.join(PROJECT_DIR, '..', '..',
-                                              'media'))
-    MEDIA_URL = '/media/'
-    STATIC_ROOT = os.path.abspath(os.path.join(PROJECT_DIR, '..', '..',
-                                               'static'))
-    STATIC_URL = '/static/'
+MEDIA_ROOT = os.path.abspath(os.path.join(PROJECT_DIR, '..', '..',
+                                          'media'))
+MEDIA_URL = '/media/'
+STATIC_ROOT = os.path.abspath(os.path.join(PROJECT_DIR, '..', '..',
+                                           'static'))
+STATIC_URL = '/static/'
 
 
 # List of finder classes that know how to find static files in

src/jane/stationxml/plugins.py

@@ -65,8 +65,23 @@ def filter_queryset_user_does_not_have_permission(self, queryset,
             pass
         elif model_type == "index":
             for restriction in restrictions:
-                queryset = queryset.exclude(json__network=restriction.network,
-                                            json__station=restriction.station)
+                kwargs = {}
+                # XXX in principle this could be handled simply by using a
+                # regex field lookup on the json field below, but in Django <
+                # 1.11 there's a bug so the regex lookup doesn't work, see
+                # django/django#6929
+                if restriction.network == '*' and restriction.station == '*':
+                    # if both network and station are '*' then all stations are
+                    # restricted
+                    return queryset.none()
+                elif restriction.network == '*':
+                    kwargs['json__station'] = restriction.station
+                elif restriction.station == '*':
+                    kwargs['json__network'] = restriction.network
+                else:
+                    kwargs['json__network'] = restriction.network
+                    kwargs['json__station'] = restriction.station
+                queryset = queryset.exclude(**kwargs)
         else:
             raise NotImplementedError()
         return queryset

src/jane/waveforms/migrations/0003_auto_20200131_1056.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.13 on 2020-01-31 10:56
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('waveforms', '0002_auto_20160706_1508'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='restriction',
+            name='network',
+            field=models.CharField(db_index=True, help_text='Use a single asterisk/star (*) to affect all station codes. Use a single asterisk/star in both fields, to restrict all network/station code combinations.', max_length=2),
+        ),
+        migrations.AlterField(
+            model_name='restriction',
+            name='station',
+            field=models.CharField(db_index=True, help_text='Use a single asterisk/star (*) to affect all station codes. Use a single asterisk/star in both fields, to restrict all network/station code combinations.', max_length=5),
+        ),
+        migrations.AlterField(
+            model_name='restriction',
+            name='users',
+            field=models.ManyToManyField(db_index=True, help_text='The restriction defined by network/station code above will apply to all users that are not added here.', to=settings.AUTH_USER_MODEL),
+        ),
+    ]

src/jane/waveforms/models.py

@@ -235,9 +235,17 @@ class Restriction(models.Model):
 
     Waveforms are generally seen as public if not listed here.
     """
-    network = models.CharField(max_length=2, db_index=True)
-    station = models.CharField(max_length=5, db_index=True)
-    users = models.ManyToManyField(User, db_index=True)
+    help_text = ('Use a single asterisk/star (*) to affect all station '
+                 'codes. Use a single asterisk/star in both fields, to '
+                 'restrict all network/station code combinations.')
+    network = models.CharField(max_length=2, db_index=True,
+                               help_text=help_text)
+    station = models.CharField(max_length=5, db_index=True,
+                               help_text=help_text)
+    users = models.ManyToManyField(
+        User, db_index=True,
+        help_text='The restriction defined by network/station code above will '
+                  'apply to all users that are not added here.')
     comment = models.TextField(blank=True, null=True)
     created_at = models.DateTimeField(auto_now_add=True, editable=False)
     created_by = models.ForeignKey(User, null=True, editable=False,