diff --git a/.travis.yml b/.travis.yml index de2d427..18a46ab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,20 +1,25 @@ language: python +dist: trusty + +sudo: false + python: - - "3.4" - "3.5" + - "3.6" env: global: secure: uH0xfi+u/FkddzkNdffIFMEv4edgHIE35no4E1nTZPCu8P3yraN9LOnfq+METCiHbFlC2O/6x32Azpg7x+jeOzfxdwCh7t5IpMg83cFV0wLZWqt940WV02LZTESjmGn5qFcJIp31pRZGxsr0S//TkfUrVNviwQ9j5kgztp7aCPfAiYYGrFEPaHuW8BiwZLVO6orAx3l/TPN3QeuF0nG0PVQiHmxbYhNTXVbjlYXRFW0swAfPBwit7yjPInqIpFbDD4slC5mhNS/EBffRE7kccPxLQAeyM69QVTkS0lHkgpEcir7NTLfKphA1uHoLtx+WutK8KwIQfzdhWOYzm0W8uVDUhAkL/p+bXT1SWMYGAtQTf6swatVjZgb0Hqb6xMXmhzMSb7vl/zhFTxkG+9TLF6edanj33j5I6wtxBUmB359Xaq9reNK1Yk6SuCAD56Yg90wHeEhgrxRURMWvjy1rV4vFPWRUx84wNdgHaTf89hiT9Vdp74eD8xqWEMgyOmgUfczJTqFcoNzfE1NKw9sycJkbuV4Sb/asjYGGadN+XTJBOAimjKj1xy3bbM2Aso6TuEMN15DE1Z9i3Ct9H9Lg9ThZvlE5EXJ3v+gP9XX81MpPyZ9pf84X1tNIiYwTzkg6JBwYiUVrf3P6M9pIKwvHSrSMos3Tn8tlIZHy4RTir/E= -services: - - postgresql - addons: - postgresql: "9.4" + postgresql: "9.6" + apt: + packages: + - postgresql-9.6-postgis-2.3 before_script: + - psql -U postgres -c "create extension postgis" - createuser --superuser test_jane before_install: @@ -26,10 +31,12 @@ before_install: - conda create --yes -n condaenv python=$TRAVIS_PYTHON_VERSION - conda install --yes -n condaenv pip - source activate condaenv - - conda install --yes -c obspy obspy psycopg2 markdown flake8 gdal pyyaml pip + - conda install --yes -c conda-forge obspy psycopg2 markdown flake8 gdal pyyaml pip - pip install codecov "django>=1.9,<1.10" djangorestframework djangorestframework-gis djangorestframework-jsonp djangorestframework-xml djangorestframework-yaml django-cors-headers django-debug-toolbar django-plugins defusedxml geojson markdown mkdocs mkdocs-bootswatch # Copy local_settings template. - cp $TRAVIS_BUILD_DIR/src/jane/local_settings.py.example $TRAVIS_BUILD_DIR/src/jane/local_settings.py + # Make sure django picks the correct geos library. + - echo -e "\n\nGDAL_LIBRARY_PATH = '/home/travis/miniconda/envs/condaenv/lib/libgdal.so'" >> $TRAVIS_BUILD_DIR/src/jane/local_settings.py install: - git version diff --git a/docs/docs/waveforms.md b/docs/docs/waveforms.md index e1d3173..a2f3bfe 100644 --- a/docs/docs/waveforms.md +++ b/docs/docs/waveforms.md @@ -145,6 +145,11 @@ per-station granularity. To do that, add a new restriction in the admin interface. As soon as a restriction has been added it will be considered protected and only users that are part of the restriction will still be able to access them. +Restrictions can also be defined with a single asterisk (`*`) in the station +(or network) code field, to make the restriction apply to all stations across a +specific network (to apply to all networks across a specific station code). Use +a single asterisk in *both* network and station code fields to add a +restriction on *all* stations. ![Add waveform restriction](./images/add_waveform_restriction.png) diff --git a/src/jane/documents/models.py b/src/jane/documents/models.py index 4c49c77..1bc2503 100644 --- a/src/jane/documents/models.py +++ b/src/jane/documents/models.py @@ -175,8 +175,9 @@ def add_or_modify_document(self, document_type, name, data, user): # Calculate the hash upfront to not upload any duplicates. sha1 = hashlib.sha1(data).hexdigest() if Document.objects.filter(sha1=sha1).exists(): - raise JaneDocumentAlreadyExists("Data already exists in the " - "database.") + raise JaneDocumentAlreadyExists( + "Data already exists in the database and document is " + "identical according to its hash.") try: document = Document.objects.get( diff --git a/src/jane/fdsnws/tests/test_station_1.py b/src/jane/fdsnws/tests/test_station_1.py index 15b479b..b350d54 100644 --- a/src/jane/fdsnws/tests/test_station_1.py +++ b/src/jane/fdsnws/tests/test_station_1.py @@ -423,6 +423,138 @@ def test_restrictions(self): **auth_headers) self.assertEqual(response.status_code, 204) + def test_restrictions_asterisk_network_and_station(self): + """ + Tests if the waveform restrictions actually work as expected. + """ + # No restrictions currently apply - we should get something. + response = self.client.get('/fdsnws/station/1/query') + self.assertEqual(response.status_code, 200) + self.assertTrue('OK' in response.reason_phrase) + inv = obspy.read_inventory(io.BytesIO(response.getvalue())) + self.assertEqual(inv.get_contents()["stations"], + ["BW.ALTM (Beilngries, Bavaria, BW-Net)"]) + + # add restriction on all stations + r = Restriction.objects.get_or_create(network="*", station="*")[0] + r.users.add(User.objects.filter(username='random')[0]) + r.save() + + # Now the same query should no longer return something as the + # station has been restricted. + response = self.client.get('/fdsnws/station/1/query') + self.assertEqual(response.status_code, 204) + + # The correct user can still get the stations. + response = self.client.get('/fdsnws/station/1/queryauth', + **self.valid_auth_headers) + self.assertEqual(response.status_code, 200) + self.assertTrue('OK' in response.reason_phrase) + inv = obspy.read_inventory(io.BytesIO(response.getvalue())) + self.assertEqual(inv.get_contents()["stations"], + ["BW.ALTM (Beilngries, Bavaria, BW-Net)"]) + + # Make another user that has not been added to this restriction - he + # should not be able to retrieve it. + self.client.logout() + User.objects.get_or_create( + username='some_dude', password=make_password('some_dude'))[0] + credentials = base64.b64encode(b'some_dude:some_dude') + auth_headers = { + 'HTTP_AUTHORIZATION': 'Basic ' + credentials.decode("ISO-8859-1") + } + response = self.client.get('/fdsnws/station/1/queryauth', + **auth_headers) + self.assertEqual(response.status_code, 204) + + def test_restrictions_asterisk_network(self): + """ + Tests if the waveform restrictions actually work as expected. + """ + # No restrictions currently apply - we should get something. + response = self.client.get('/fdsnws/station/1/query') + self.assertEqual(response.status_code, 200) + self.assertTrue('OK' in response.reason_phrase) + inv = obspy.read_inventory(io.BytesIO(response.getvalue())) + self.assertEqual(inv.get_contents()["stations"], + ["BW.ALTM (Beilngries, Bavaria, BW-Net)"]) + + # add restriction on ALTM stations + r = Restriction.objects.get_or_create(network="*", station="ALTM")[0] + r.users.add(User.objects.filter(username='random')[0]) + r.save() + + # Now the same query should no longer return something as the + # station has been restricted. + response = self.client.get('/fdsnws/station/1/query') + self.assertEqual(response.status_code, 204) + + # The correct user can still get the stations. + response = self.client.get('/fdsnws/station/1/queryauth', + **self.valid_auth_headers) + self.assertEqual(response.status_code, 200) + self.assertTrue('OK' in response.reason_phrase) + inv = obspy.read_inventory(io.BytesIO(response.getvalue())) + self.assertEqual(inv.get_contents()["stations"], + ["BW.ALTM (Beilngries, Bavaria, BW-Net)"]) + + # Make another user that has not been added to this restriction - he + # should not be able to retrieve it. + self.client.logout() + User.objects.get_or_create( + username='some_dude', password=make_password('some_dude'))[0] + credentials = base64.b64encode(b'some_dude:some_dude') + auth_headers = { + 'HTTP_AUTHORIZATION': 'Basic ' + credentials.decode("ISO-8859-1") + } + response = self.client.get('/fdsnws/station/1/queryauth', + **auth_headers) + self.assertEqual(response.status_code, 204) + + def test_restrictions_asterisk_station(self): + """ + Tests if the waveform restrictions actually work as expected. + """ + # No restrictions currently apply - we should get something. + response = self.client.get('/fdsnws/station/1/query') + self.assertEqual(response.status_code, 200) + self.assertTrue('OK' in response.reason_phrase) + inv = obspy.read_inventory(io.BytesIO(response.getvalue())) + self.assertEqual(inv.get_contents()["stations"], + ["BW.ALTM (Beilngries, Bavaria, BW-Net)"]) + + # add restriction on all BW-network stations + r = Restriction.objects.get_or_create(network="BW", station="*")[0] + r.users.add(User.objects.filter(username='random')[0]) + r.save() + + # Now the same query should no longer return something as the + # station has been restricted. + response = self.client.get('/fdsnws/station/1/query') + self.assertEqual(response.status_code, 204) + + # The correct user can still get the stations. + response = self.client.get('/fdsnws/station/1/queryauth', + **self.valid_auth_headers) + self.assertEqual(response.status_code, 200) + self.assertTrue('OK' in response.reason_phrase) + inv = obspy.read_inventory(io.BytesIO(response.getvalue())) + self.assertEqual(inv.get_contents()["stations"], + ["BW.ALTM (Beilngries, Bavaria, BW-Net)"]) + + # Make another user that has not been added to this restriction - he + # should not be able to retrieve it. + self.client.logout() + User.objects.get_or_create( + username='some_dude', password=make_password('some_dude'))[0] + credentials = base64.b64encode(b'some_dude:some_dude') + auth_headers = { + 'HTTP_AUTHORIZATION': 'Basic ' + credentials.decode("ISO-8859-1") + } + response = self.client.get('/fdsnws/station/1/queryauth', + **auth_headers) + self.assertEqual(response.status_code, 204) + class Station1LiveServerTestCase(LiveServerTestCase): """ diff --git a/src/jane/settings.py b/src/jane/settings.py index 2cf9307..c1894c4 100644 --- a/src/jane/settings.py +++ b/src/jane/settings.py @@ -32,18 +32,12 @@ DATE_FORMAT = "Y-m-d" -if DEPLOYED: - MEDIA_ROOT = '/home/django/www/media' - MEDIA_URL = '/media/' - STATIC_ROOT = '/home/django/www/static' - STATIC_URL = '/static/' -else: - MEDIA_ROOT = os.path.abspath(os.path.join(PROJECT_DIR, '..', '..', - 'media')) - MEDIA_URL = '/media/' - STATIC_ROOT = os.path.abspath(os.path.join(PROJECT_DIR, '..', '..', - 'static')) - STATIC_URL = '/static/' +MEDIA_ROOT = os.path.abspath(os.path.join(PROJECT_DIR, '..', '..', + 'media')) +MEDIA_URL = '/media/' +STATIC_ROOT = os.path.abspath(os.path.join(PROJECT_DIR, '..', '..', + 'static')) +STATIC_URL = '/static/' # List of finder classes that know how to find static files in diff --git a/src/jane/stationxml/plugins.py b/src/jane/stationxml/plugins.py index f216983..37642c9 100644 --- a/src/jane/stationxml/plugins.py +++ b/src/jane/stationxml/plugins.py @@ -65,8 +65,23 @@ def filter_queryset_user_does_not_have_permission(self, queryset, pass elif model_type == "index": for restriction in restrictions: - queryset = queryset.exclude(json__network=restriction.network, - json__station=restriction.station) + kwargs = {} + # XXX in principle this could be handled simply by using a + # regex field lookup on the json field below, but in Django < + # 1.11 there's a bug so the regex lookup doesn't work, see + # django/django#6929 + if restriction.network == '*' and restriction.station == '*': + # if both network and station are '*' then all stations are + # restricted + return queryset.none() + elif restriction.network == '*': + kwargs['json__station'] = restriction.station + elif restriction.station == '*': + kwargs['json__network'] = restriction.network + else: + kwargs['json__network'] = restriction.network + kwargs['json__station'] = restriction.station + queryset = queryset.exclude(**kwargs) else: raise NotImplementedError() return queryset diff --git a/src/jane/waveforms/migrations/0003_auto_20200131_1056.py b/src/jane/waveforms/migrations/0003_auto_20200131_1056.py new file mode 100644 index 0000000..407c0c4 --- /dev/null +++ b/src/jane/waveforms/migrations/0003_auto_20200131_1056.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.13 on 2020-01-31 10:56 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('waveforms', '0002_auto_20160706_1508'), + ] + + operations = [ + migrations.AlterField( + model_name='restriction', + name='network', + field=models.CharField(db_index=True, help_text='Use a single asterisk/star (*) to affect all station codes. Use a single asterisk/star in both fields, to restrict all network/station code combinations.', max_length=2), + ), + migrations.AlterField( + model_name='restriction', + name='station', + field=models.CharField(db_index=True, help_text='Use a single asterisk/star (*) to affect all station codes. Use a single asterisk/star in both fields, to restrict all network/station code combinations.', max_length=5), + ), + migrations.AlterField( + model_name='restriction', + name='users', + field=models.ManyToManyField(db_index=True, help_text='The restriction defined by network/station code above will apply to all users that are not added here.', to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/src/jane/waveforms/models.py b/src/jane/waveforms/models.py index caae726..2849693 100644 --- a/src/jane/waveforms/models.py +++ b/src/jane/waveforms/models.py @@ -235,9 +235,17 @@ class Restriction(models.Model): Waveforms are generally seen as public if not listed here. """ - network = models.CharField(max_length=2, db_index=True) - station = models.CharField(max_length=5, db_index=True) - users = models.ManyToManyField(User, db_index=True) + help_text = ('Use a single asterisk/star (*) to affect all station ' + 'codes. Use a single asterisk/star in both fields, to ' + 'restrict all network/station code combinations.') + network = models.CharField(max_length=2, db_index=True, + help_text=help_text) + station = models.CharField(max_length=5, db_index=True, + help_text=help_text) + users = models.ManyToManyField( + User, db_index=True, + help_text='The restriction defined by network/station code above will ' + 'apply to all users that are not added here.') comment = models.TextField(blank=True, null=True) created_at = models.DateTimeField(auto_now_add=True, editable=False) created_by = models.ForeignKey(User, null=True, editable=False,