From cac0710593bd9b44ad0f627c5762fb860dddb05f Mon Sep 17 00:00:00 2001 From: Oliver Stolpe Date: Wed, 23 Oct 2024 17:32:44 +0200 Subject: [PATCH] feat: upgrade to sodar core v1 (#170) --- .github/workflows/build.yml | 8 +- Makefile | 5 + config/settings/base.py | 109 +++++++----------- config/urls.py | 82 ++++++------- config/wsgi.py | 1 + ...012_alter_containerbackgroundjob_bg_job.py | 25 ++++ containers/models.py | 6 +- containers/statemachines.py | 8 +- containers/tests/helpers.py | 1 + containers/tests/test_models.py | 1 + containers/tests/test_permissions.py | 6 +- containers/tests/test_permissions_api.py | 9 +- containers/tests/test_tasks.py | 1 + containers/tests/test_templatetags.py | 1 + containers/tests/test_views.py | 7 +- containers/tests/test_views_api.py | 18 ++- containers/urls.py | 86 +++++++------- containertemplates/tests/helpers.py | 1 + containertemplates/tests/test_permissions.py | 6 +- containertemplates/tests/test_views.py | 5 +- containertemplates/urls.py | 58 +++++----- containertemplates/views.py | 8 +- docker/Dockerfile | 8 +- kioscadmin/management/commands/stop_all.py | 1 + kioscadmin/urls.py | 6 +- requirements/base.txt | 77 +++++++------ requirements/local.txt | 9 +- requirements/production.txt | 8 +- requirements/test.txt | 25 ++-- utility/install_os_dependencies.sh | 4 - 30 files changed, 293 insertions(+), 297 deletions(-) create mode 100644 containers/migrations/0012_alter_containerbackgroundjob_bg_job.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6d01b95..b3fe256 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,12 +7,12 @@ jobs: strategy: matrix: python-version: - - '3.8' - '3.9' - '3.10' + - '3.11' services: postgres: - image: postgres:11 + image: postgres:16 env: POSTGRES_DB: kiosc POSTGRES_USER: kiosc @@ -42,7 +42,7 @@ jobs: uses: actions/checkout@v2 - name: Install project Python dependencies run: | - pip install wheel==0.37.1 + pip install wheel==0.42.0 pip install -r requirements/local.txt pip install -r requirements/test.txt - name: Download icons @@ -63,4 +63,4 @@ jobs: with: project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} coverage-reports: coverage.xml - if: ${{ matrix.python-version == '3.8' }} + if: ${{ matrix.python-version == '3.11' }} diff --git a/Makefile b/Makefile index 946cc89..ebf6723 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,11 @@ serve: $(MANAGE) runserver --settings=config.settings.local +.PHONY: asgi +asgi: + python -m uvicorn config.asgi:application + + .PHONY: serve_target serve_target: $(MANAGE) runserver 0.0.0.0:$(target_port) --settings=config.settings.local_target diff --git a/config/settings/base.py b/config/settings/base.py index 8c12317..5887151 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -7,6 +7,7 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/dev/ref/settings/ """ + import os import environ @@ -59,6 +60,7 @@ "markupfield", # For markdown "rest_framework", # For API views "knox", # For token auth + "social_django", # For OIDC authentication "docs", # For the online user documentation/manual "dal", # For user search combo box "dal_select2", @@ -282,7 +284,7 @@ AUTOSLUG_SLUGIFY_FUNCTION = "slugify.slugify" # Location of root django.contrib.admin URL, use {% url 'admin:index' %} -ADMIN_URL = r"^admin/" +ADMIN_URL = "admin/" # Celery configuration (for background jobs) # ------------------------------------------------------------------------------ @@ -361,9 +363,9 @@ AUTH_LDAP_CA_CERT_FILE = env.str("AUTH_LDAP_CA_CERT_FILE", None) AUTH_LDAP_CONNECTION_OPTIONS = LDAP_DEFAULT_CONN_OPTIONS if AUTH_LDAP_CA_CERT_FILE: - AUTH_LDAP_CONNECTION_OPTIONS[ - ldap.OPT_X_TLS_CACERTFILE - ] = AUTH_LDAP_CA_CERT_FILE + AUTH_LDAP_CONNECTION_OPTIONS[ldap.OPT_X_TLS_CACERTFILE] = ( + AUTH_LDAP_CA_CERT_FILE + ) AUTH_LDAP_CONNECTION_OPTIONS[ldap.OPT_X_TLS_NEWCTX] = 0 AUTH_LDAP_USER_SEARCH = LDAPSearch( env.str("AUTH_LDAP_USER_SEARCH_BASE", None), @@ -392,9 +394,9 @@ AUTH_LDAP2_CA_CERT_FILE = env.str("AUTH_LDAP2_CA_CERT_FILE", None) AUTH_LDAP2_CONNECTION_OPTIONS = LDAP_DEFAULT_CONN_OPTIONS if AUTH_LDAP2_CA_CERT_FILE: - AUTH_LDAP2_CONNECTION_OPTIONS[ - ldap.OPT_X_TLS_CACERTFILE - ] = AUTH_LDAP2_CA_CERT_FILE + AUTH_LDAP2_CONNECTION_OPTIONS[ldap.OPT_X_TLS_CACERTFILE] = ( + AUTH_LDAP2_CA_CERT_FILE + ) AUTH_LDAP2_CONNECTION_OPTIONS[ldap.OPT_X_TLS_NEWCTX] = 0 AUTH_LDAP2_USER_SEARCH = LDAPSearch( @@ -416,71 +418,40 @@ ) -# SAML configuration +# OpenID Connect (OIDC) configuration # ------------------------------------------------------------------------------ +ENABLE_OIDC = env.bool("ENABLE_OIDC", False) -ENABLE_SAML = env.bool("ENABLE_SAML", False) -SAML2_AUTH = { - # Required setting - "SAML_CLIENT_SETTINGS": { # Pysaml2 Saml client settings (https://pysaml2.readthedocs.io/en/latest/howto/config.html) - "entityid": env.str( - "SAML_CLIENT_ENTITY_ID", "SODARcore" - ), # The optional entity ID string to be passed in the 'Issuer' element of authn request, if required by the IDP. - "entitybaseurl": env.str( - "SAML_CLIENT_ENTITY_URL", "https://localhost:8000" - ), - "metadata": { - "local": [ - env.str( - "SAML_CLIENT_METADATA_FILE", "metadata.xml" - ), # The auto(dynamic) metadata configuration URL of SAML2 - ], - }, - "service": { - "sp": { - "idp": env.str( - "SAML_CLIENT_IPD", - "https://sso.hpc.bihealth.org/auth/realms/cubi", - ), - # Keycloak expects client signature - "authn_requests_signed": "true", - # Enforce POST binding which is required by keycloak - "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST", - }, - }, - "key_file": env.str("SAML_CLIENT_KEY_FILE", "key.pem"), - "cert_file": env.str("SAML_CLIENT_CERT_FILE", "cert.pem"), - "xmlsec_binary": env.str("SAML_CLIENT_XMLSEC1", "/usr/bin/xmlsec1"), - "encryption_keypairs": [ - { - "key_file": env.str("SAML_CLIENT_KEY_FILE", "key.pem"), - "cert_file": env.str("SAML_CLIENT_CERT_FILE", "cert.pem"), - } - ], - }, - "DEFAULT_NEXT_URL": "/", # Custom target redirect URL after the user get logged in. Default to /admin if not set. This setting will be overwritten if you have parameter ?next= specificed in the login URL. - # # Optional settings below - # 'NEW_USER_PROFILE': { - # 'USER_GROUPS': [], # The default group name when a new user logs in - # 'ACTIVE_STATUS': True, # The default active status for new users - # 'STAFF_STATUS': True, # The staff status for new users - # 'SUPERUSER_STATUS': False, # The superuser status for new users - # }, - # 'ATTRIBUTES_MAP': { # Change Email/UserName/FirstName/LastName to corresponding SAML2 userprofile attributes. - # 'email': 'Email', - # 'username': 'UserName', - # 'first_name': 'FirstName', - # 'last_name': 'LastName', - # }, - # 'TRIGGER': { - # 'FIND_USER': 'path.to.your.find.user.hook.method', - # 'NEW_USER': 'path.to.your.new.user.hook.method', - # 'CREATE_USER': 'path.to.your.create.user.hook.method', - # 'BEFORE_LOGIN': 'path.to.your.login.hook.method', - # }, - # 'ASSERTION_URL': 'https://your.url.here', # Custom URL to validate incoming SAML requests against -} +if ENABLE_OIDC: + AUTHENTICATION_BACKENDS = tuple( + itertools.chain( + ("social_core.backends.open_id_connect.OpenIdConnectAuth",), + AUTHENTICATION_BACKENDS, + ) + ) + TEMPLATES[0]["OPTIONS"]["context_processors"] += [ + "social_django.context_processors.backends", + "social_django.context_processors.login_redirect", + ] + SOCIAL_AUTH_JSONFIELD_ENABLED = True + SOCIAL_AUTH_JSONFIELD_CUSTOM = "django.db.models.JSONField" + SOCIAL_AUTH_USER_MODEL = AUTH_USER_MODEL + SOCIAL_AUTH_ADMIN_USER_SEARCH_FIELDS = [ + "username", + "name", + "first_name", + "last_name", + "email", + ] + SOCIAL_AUTH_OIDC_OIDC_ENDPOINT = env.str( + "SOCIAL_AUTH_OIDC_OIDC_ENDPOINT", None + ) + SOCIAL_AUTH_OIDC_KEY = env.str("SOCIAL_AUTH_OIDC_KEY", "CHANGEME") + SOCIAL_AUTH_OIDC_SECRET = env.str("SOCIAL_AUTH_OIDC_SECRET", "CHANGEME") + SOCIAL_AUTH_OIDC_USERNAME_KEY = env.str( + "SOCIAL_AUTH_OIDC_USERNAME_KEY", "username" + ) # Logging diff --git a/config/urls.py b/config/urls.py index 3342792..d0de0a3 100644 --- a/config/urls.py +++ b/config/urls.py @@ -1,5 +1,6 @@ from django.conf import settings -from django.conf.urls import include, url +from django.conf.urls import include +from django.urls import path from django.conf.urls.static import static from django.contrib import admin from django.contrib.auth import views as auth_views @@ -7,97 +8,80 @@ from django.views import defaults as default_views from django.views.generic import TemplateView -import django_saml2_auth.views - from projectroles.views import HomeView urlpatterns = [ - url(r"^$", HomeView.as_view(), name="home"), - url( - r"^about/$", + path("", HomeView.as_view(), name="home"), + path( + "about/", TemplateView.as_view(template_name="pages/about.html"), name="about", ), # Admin URLs - most occur before Django Admin, otherwise urls will be matched by that. - url(r"^kioscadmin/", include("kioscadmin.urls")), + path("kioscadmin/", include("kioscadmin.urls")), # Django Admin, use {% url 'admin:index' %} - url(settings.ADMIN_URL, admin.site.urls), + path(settings.ADMIN_URL, admin.site.urls), # Login and logout - url( - r"^login/$", + path( + "login/", auth_views.LoginView.as_view(template_name="users/login.html"), name="login", ), - url(r"^logout/$", auth_views.logout_then_login, name="logout"), + path("logout/", auth_views.logout_then_login, name="logout"), # Auth - url(r"api/auth/", include("knox.urls")), + path("api/auth/", include("knox.urls")), # Projectroles URLs - url(r"^project/", include("projectroles.urls")), + path("project/", include("projectroles.urls")), # Timeline URLs - url(r"^timeline/", include("timeline.urls")), + path("timeline/", include("timeline.urls")), # django-db-file-storage URLs (obfuscated for users) # TODO: Change the URL to something obfuscated (e.g. random string) - url(r"^CHANGE-ME/", include("db_file_storage.urls")), + path("CHANGE-ME/", include("db_file_storage.urls")), # Background Jobs URLs - url(r"^bgjobs/", include("bgjobs.urls")), + path("bgjobs/", include("bgjobs.urls")), # Data Cache app - # url(r'^cache/', include('sodarcache.urls')), + # path(r'^cache/', include('sodarcache.urls')), # User Profile URLs - url(r"^user/", include("userprofile.urls")), + path("user/", include("userprofile.urls")), # Admin Alerts URLs - url(r"^adminalerts/", include("adminalerts.urls")), + path("adminalerts/", include("adminalerts.urls")), # App Alerts URLs - url("^appalerts/", include("appalerts.urls")), + path("appalerts/", include("appalerts.urls")), # Site Info URLs - url(r"^siteinfo/", include("siteinfo.urls")), + path("siteinfo/", include("siteinfo.urls")), # API Tokens URLs - url(r"^tokens/", include("tokens.urls")), + path("tokens/", include("tokens.urls")), # Containers URLs - url(r"^containers/", include("containers.urls")), + path("containers/", include("containers.urls")), # Containertemplates URLs - url(r"^containertemplates/", include("containertemplates.urls")), + path("containertemplates/", include("containertemplates.urls")), # Iconify icon URLs - url(r"^icons/", include("dj_iconify.urls")), - # These are the SAML2 related URLs. You can change "^saml2_auth/" regex to - # any path you want, like "^sso_auth/", "^sso_login/", etc. (required) - # url(r'^saml2_auth/', include('django_saml2_auth.urls')), - # The following line will replace the default user login with SAML2 (optional) - # If you want to specific the after-login-redirect-URL, use parameter "?next=/the/path/you/want" - # with this view. - # url(r'^sso/login/$', django_saml2_auth.views.signin), - # The following line will replace the admin login with SAML2 (optional) - # If you want to specific the after-login-redirect-URL, use parameter "?next=/the/path/you/want" - # with this view. - # url(r'^sso/admin/login/$', django_saml2_auth.views.signin), - # The following line will replace the default user logout with the signout page (optional) - # url(r'^sso/logout/$', django_saml2_auth.views.signout), - # The following line will replace the default admin user logout with the signout page (optional) - # url(r'^sso/admin/logout/$', django_saml2_auth.views.signout), + path("icons/", include("dj_iconify.urls")), ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) if settings.KIOSC_EMBEDDED_FILES: - urlpatterns.append(url(r"^files/", include("filesfolders.urls"))) + urlpatterns.append(path("files/", include("filesfolders.urls"))) if settings.DEBUG: # This allows the error pages to be debugged during development, just visit # these url in browser to see how these error pages look like. urlpatterns += [ - url( - r"^400/$", + path( + "400/", default_views.bad_request, kwargs={"exception": Exception("Bad Request!")}, ), - url( - r"^403/$", + path( + "403/", default_views.permission_denied, kwargs={"exception": Exception("Permission Denied")}, ), - url( - r"^404/$", + path( + "404/", default_views.page_not_found, kwargs={"exception": Exception("Page not Found")}, ), - url(r"^500/$", default_views.server_error), + path("500/", default_views.server_error), ] urlpatterns += staticfiles_urlpatterns() @@ -106,5 +90,5 @@ import debug_toolbar urlpatterns = [ - url(r"^__debug__/", include(debug_toolbar.urls)) + path("__debug__/", include(debug_toolbar.urls)) ] + urlpatterns diff --git a/config/wsgi.py b/config/wsgi.py index b237fca..300163c 100644 --- a/config/wsgi.py +++ b/config/wsgi.py @@ -13,6 +13,7 @@ framework. """ + import os import sys diff --git a/containers/migrations/0012_alter_containerbackgroundjob_bg_job.py b/containers/migrations/0012_alter_containerbackgroundjob_bg_job.py new file mode 100644 index 0000000..f0b3790 --- /dev/null +++ b/containers/migrations/0012_alter_containerbackgroundjob_bg_job.py @@ -0,0 +1,25 @@ +# Generated by Django 4.2.16 on 2024-10-23 15:47 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("bgjobs", "0001_squashed_0006_auto_20200526_1657"), + ("containers", "0011_alter_container_container_path"), + ] + + operations = [ + migrations.AlterField( + model_name="containerbackgroundjob", + name="bg_job", + field=models.ForeignKey( + help_text="Background job for state etc.", + on_delete=django.db.models.deletion.CASCADE, + related_name="%(app_label)s_%(class)s_related", + to="bgjobs.backgroundjob", + ), + ), + ] diff --git a/containers/models.py b/containers/models.py index 598aedf..4f6d9c1 100644 --- a/containers/models.py +++ b/containers/models.py @@ -469,9 +469,9 @@ def merge_order(self, *args, **kwargs): return sorted( qs, - key=lambda a: a.date_docker_log - if a.date_docker_log - else a.date_created, + key=lambda a: ( + a.date_docker_log if a.date_docker_log else a.date_created + ), ) def get_logs_as_str(self, *args, **kwargs): diff --git a/containers/statemachines.py b/containers/statemachines.py index 31e75fd..a2aaa53 100644 --- a/containers/statemachines.py +++ b/containers/statemachines.py @@ -441,9 +441,11 @@ def on_pull(self): detach=True, image=self.container.image_id, environment=environment, - command=shlex.split(self.container.command) - if self.container.command - else None, + command=( + shlex.split(self.container.command) + if self.container.command + else None + ), ports=[self.container.container_port], host_config=self.cli.create_host_config( ulimits=[ diff --git a/containers/tests/helpers.py b/containers/tests/helpers.py index c066cc6..a3957e9 100644 --- a/containers/tests/helpers.py +++ b/containers/tests/helpers.py @@ -1,4 +1,5 @@ """Helpers for the container tests.""" + import uuid import dateutil.parser from django.conf import settings diff --git a/containers/tests/test_models.py b/containers/tests/test_models.py index e68c29a..23eeaad 100644 --- a/containers/tests/test_models.py +++ b/containers/tests/test_models.py @@ -1,4 +1,5 @@ """Tests for the container models""" + import json from datetime import timedelta diff --git a/containers/tests/test_permissions.py b/containers/tests/test_permissions.py index 2ed2338..eebb930 100644 --- a/containers/tests/test_permissions.py +++ b/containers/tests/test_permissions.py @@ -1,8 +1,9 @@ """Permission tests.""" + from unittest.mock import patch from django.urls import reverse -from projectroles.tests.test_permissions import TestProjectPermissionBase +from projectroles.tests.test_permissions import ProjectPermissionTestBase from urllib3_mock import Responses from containers.models import STATE_RUNNING @@ -12,7 +13,7 @@ responses = Responses("requests.packages.urllib3") -class TestContainerPermissions(TestProjectPermissionBase): +class TestContainerPermissions(ProjectPermissionTestBase): """Test permissions for container app.""" def setUp(self): @@ -271,6 +272,7 @@ def test_container_unpause(self, mock): self.assert_response(url, bad_users, 302) mock.assert_called() + # urllib3-mock not working with Python 3.11+ :-/ @responses.activate def test_proxy(self): """Test permissions for the ``proxy`` view.""" diff --git a/containers/tests/test_permissions_api.py b/containers/tests/test_permissions_api.py index b25f7c5..de23715 100644 --- a/containers/tests/test_permissions_api.py +++ b/containers/tests/test_permissions_api.py @@ -1,21 +1,18 @@ """Permission tests.""" + from unittest.mock import patch from django.urls import reverse from containers.models import Container -from projectroles.tests.test_permissions_api import TestProjectAPIPermissionBase +from projectroles.tests.test_permissions_api import ProjectAPIPermissionTestBase from rest_framework import status -from urllib3_mock import Responses from containers.tests.factories import ContainerFactory from django.test import override_settings -responses = Responses("requests.packages.urllib3") - - -class TestContainerAPIPermissions(TestProjectAPIPermissionBase): +class TestContainerAPIPermissions(ProjectAPIPermissionTestBase): """Test API permissions for container app.""" def setUp(self): diff --git a/containers/tests/test_tasks.py b/containers/tests/test_tasks.py index 8ddcbf9..17d61f5 100644 --- a/containers/tests/test_tasks.py +++ b/containers/tests/test_tasks.py @@ -1,4 +1,5 @@ """Test container tasks.""" + import time from unittest.mock import patch, call diff --git a/containers/tests/test_templatetags.py b/containers/tests/test_templatetags.py index 093129e..ef2d188 100644 --- a/containers/tests/test_templatetags.py +++ b/containers/tests/test_templatetags.py @@ -1,4 +1,5 @@ """Tests for the ``templatetags`` module.""" + import json from test_plus.test import TestCase diff --git a/containers/tests/test_views.py b/containers/tests/test_views.py index 07319a7..8598a50 100644 --- a/containers/tests/test_views.py +++ b/containers/tests/test_views.py @@ -1,4 +1,5 @@ """Tests for the container views.""" + import json from unittest.mock import patch @@ -456,9 +457,9 @@ def test_post_success_updated_initial_mode_host_masked_environment(self): self.container1.environment_secret_keys = "secret,secret_to_update" self.container1.save() self.post_data_host["environment"] = json.dumps(environment_post) - self.post_data_host[ - "environment_secret_keys" - ] = self.container1.environment_secret_keys + self.post_data_host["environment_secret_keys"] = ( + self.container1.environment_secret_keys + ) with self.login(self.superuser): response = self.client.post( diff --git a/containers/tests/test_views_api.py b/containers/tests/test_views_api.py index 62362e8..26ad53f 100644 --- a/containers/tests/test_views_api.py +++ b/containers/tests/test_views_api.py @@ -1,8 +1,8 @@ """Tests for the container API views.""" + from unittest.mock import patch from rest_framework import status -from urllib3_mock import Responses from django.forms import model_to_dict from django.urls import reverse @@ -19,12 +19,10 @@ from containers.tests.helpers import TestContainerCreationMixin from containers.views import CELERY_SUBMIT_COUNTDOWN from projectroles.models import Project -from projectroles.tests.test_views_api import TestAPIViewsBase - -responses = Responses("requests.packages.urllib3") +from projectroles.tests.test_views_api import APIViewTestBase -class TestContainerListAPIView(TestContainerCreationMixin, TestAPIViewsBase): +class TestContainerListAPIView(TestContainerCreationMixin, APIViewTestBase): """Tests for ``ContainerListAPIView``.""" def test_get_success_list_empty(self): @@ -97,7 +95,7 @@ def test_get_success_list_two_items(self): self.assertEqual(response.json(), [container2, container1]) -class TestContainerCreateAPIView(TestContainerCreationMixin, TestAPIViewsBase): +class TestContainerCreateAPIView(TestContainerCreationMixin, APIViewTestBase): """Tests for ``ContainerCreateAPIView``.""" def setUp(self): @@ -190,7 +188,7 @@ def test_post_success_all_fields(self): self.assertDictEqual(result, self.post_data_all) -class TestContainerDeleteAPIView(TestContainerCreationMixin, TestAPIViewsBase): +class TestContainerDeleteAPIView(TestContainerCreationMixin, APIViewTestBase): """Tests for ``ContainerDeleteAPIView``.""" def setUp(self): @@ -249,7 +247,7 @@ def test_delete_non_existent(self): self.assertEqual(Container.objects.count(), 1) -class TestContainerDetailAPIView(TestContainerCreationMixin, TestAPIViewsBase): +class TestContainerDetailAPIView(TestContainerCreationMixin, APIViewTestBase): """Tests for ``ContainerDetailAPIView``.""" def setUp(self): @@ -291,7 +289,7 @@ def test_get_non_existent(self): self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) -class TestContainerStartAPIView(TestContainerCreationMixin, TestAPIViewsBase): +class TestContainerStartAPIView(TestContainerCreationMixin, APIViewTestBase): """Tests for ``ContainerStartAPIView``.""" def setUp(self): @@ -330,7 +328,7 @@ def test_get_non_existent(self): self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) -class TestContainerStopAPIView(TestContainerCreationMixin, TestAPIViewsBase): +class TestContainerStopAPIView(TestContainerCreationMixin, APIViewTestBase): """Tests for ``ContainerStopAPIView``.""" def setUp(self): diff --git a/containers/urls.py b/containers/urls.py index a57cfe5..d1839e0 100644 --- a/containers/urls.py +++ b/containers/urls.py @@ -1,4 +1,4 @@ -from django.conf.urls import url +from django.urls import path, re_path from django.views.decorators.csrf import csrf_exempt from . import views, consumers, views_api @@ -7,115 +7,115 @@ ui_urlpatterns = [ - url( - regex=r"^(?P[0-9a-f-]+)$", + path( + "", view=views.ContainerListView.as_view(), name="list", ), - url( - regex=r"^detail/(?P[0-9a-f-]+)$", + path( + "detail/", view=views.ContainerDetailView.as_view(), name="detail", ), - url( - regex=r"^create/(?P[0-9a-f-]+)$", + path( + "create/", view=views.ContainerCreateView.as_view(), name="create", ), - url( - regex=r"^update/(?P[0-9a-f-]+)$", + path( + "update/", view=views.ContainerUpdateView.as_view(), name="update", ), - url( - regex=r"^delete/(?P[0-9a-f-]+)$", + path( + "delete/", view=views.ContainerDeleteView.as_view(), name="delete", ), - url( - regex=r"^start/(?P[0-9a-f-]+)$", + path( + "start/", view=views.ContainerStartView.as_view(), name="start", ), - url( - regex=r"^stop/(?P[0-9a-f-]+)$", + path( + "stop/", view=views.ContainerStopView.as_view(), name="stop", ), - url( - regex=r"^pause/(?P[0-9a-f-]+)$", + path( + "pause/", view=views.ContainerPauseView.as_view(), name="pause", ), - url( - regex=r"^unpause/(?P[0-9a-f-]+)$", + path( + "unpause/", view=views.ContainerUnpauseView.as_view(), name="unpause", ), - url( - regex=r"^restart/(?P[0-9a-f-]+)$", + path( + "restart/", view=views.ContainerRestartView.as_view(), name="restart", ), - url( - regex=r"^proxy/(?P[0-9a-f-]+)/(?P.*)$", + re_path( + r"^proxy/(?P[0-9a-f-]+)/(?P.*)$", view=csrf_exempt(views.ReverseProxyView.as_view()), name="proxy", ), - url( - regex=r"^proxy/lobby/(?P[0-9a-f-]+)$", + path( + "proxy/lobby/", view=views.ContainerProxyLobbyView.as_view(), name="proxy-lobby", ), - url( - regex=r"^file/serve/(?P[0-9a-f-]+)/(?P.*)$", + path( + "file/serve//", view=views.FileServeView.as_view(), name="file-serve", ), # Ajax views - url( - regex=r"^ajax/get-dynamic-details/(?P[0-9a-f-]+)$", + path( + "ajax/get-dynamic-details/", view=views.ContainerGetDynamicDetailsApiView.as_view(), name="ajax-get-dynamic-details", ), ] api_urlpatterns = [ - url( - regex=r"^api/(?P[0-9a-f-]+)$", + path( + "api/", view=views_api.ContainerListAPIView.as_view(), name="api-list", ), - url( - regex=r"^api/detail/(?P[0-9a-f-]+)$", + path( + "api/detail/", view=views_api.ContainerDetailAPIView.as_view(), name="api-detail", ), - url( - regex=r"^api/create/(?P[0-9a-f-]+)$", + path( + "api/create/", view=views_api.ContainerCreateAPIView.as_view(), name="api-create", ), - url( - regex=r"^api/delete/(?P[0-9a-f-]+)$", + path( + "api/delete/", view=views_api.ContainerDeleteAPIView.as_view(), name="api-delete", ), - url( - regex=r"^api/start/(?P[0-9a-f-]+)$", + path( + "api/start/", view=views_api.ContainerStartAPIView.as_view(), name="api-start", ), - url( - regex=r"^api/stop/(?P[0-9a-f-]+)$", + path( + "api/stop/", view=views_api.ContainerStopAPIView.as_view(), name="api-stop", ), ] websocket_urlpatterns = [ - url( - (r"^containers/proxy/(?P[0-9a-f-]+)/(?P.*)$"), + re_path( + r"container/proxy/(?P[0-9a-f-]+)/(?P.*)$", consumers.TunnelConsumer, ) ] diff --git a/containertemplates/tests/helpers.py b/containertemplates/tests/helpers.py index 9df359a..a3bebf3 100644 --- a/containertemplates/tests/helpers.py +++ b/containertemplates/tests/helpers.py @@ -1,4 +1,5 @@ """Helpers for the container tests.""" + import uuid from projectroles.forms import ( diff --git a/containertemplates/tests/test_permissions.py b/containertemplates/tests/test_permissions.py index b6e3675..b4571b6 100644 --- a/containertemplates/tests/test_permissions.py +++ b/containertemplates/tests/test_permissions.py @@ -1,7 +1,7 @@ """Permission tests.""" from django.urls import reverse -from projectroles.tests.test_permissions import TestProjectPermissionBase +from projectroles.tests.test_permissions import ProjectPermissionTestBase from containertemplates.tests.factories import ( ContainerTemplateSiteFactory, @@ -9,7 +9,7 @@ ) -class TestContainerTemplateSitePermissions(TestProjectPermissionBase): +class TestContainerTemplateSitePermissions(ProjectPermissionTestBase): """Test permissions for site-wide containertemplates app.""" def setUp(self): @@ -152,7 +152,7 @@ def test_containertemplatesite_duplicate(self): self.assert_response(url, bad_users, 302) -class TestContainerTemplateProjectPermissions(TestProjectPermissionBase): +class TestContainerTemplateProjectPermissions(ProjectPermissionTestBase): """Test permissions for project-wide containertemplates app.""" def setUp(self): diff --git a/containertemplates/tests/test_views.py b/containertemplates/tests/test_views.py index 0538813..97e57eb 100644 --- a/containertemplates/tests/test_views.py +++ b/containertemplates/tests/test_views.py @@ -1,8 +1,8 @@ """Tests for the containertemplate views.""" + import json from django.contrib.messages import get_messages -from urllib3_mock import Responses from django.forms import model_to_dict from django.urls import reverse @@ -15,9 +15,6 @@ from containertemplates.tests.helpers import TestBase -responses = Responses("requests.packages.urllib3") - - class TestContainerTemplateSiteListView(TestBase): """Tests for ``ContainerTemplateSiteListView``.""" diff --git a/containertemplates/urls.py b/containertemplates/urls.py index 3c3c88a..e32af29 100644 --- a/containertemplates/urls.py +++ b/containertemplates/urls.py @@ -1,4 +1,4 @@ -from django.conf.urls import url +from django.urls import path from . import views @@ -7,74 +7,74 @@ urlpatterns = [ - url( - regex=r"^site$", + path( + "site", view=views.ContainerTemplateSiteListView.as_view(), name="site-list", ), - url( - regex=r"^site/detail/(?P[0-9a-f-]+)$", + path( + "site/detail/", view=views.ContainerTemplateSiteDetailView.as_view(), name="site-detail", ), - url( - regex=r"^site/create$", + path( + "site/create", view=views.ContainerTemplateSiteCreateView.as_view(), name="site-create", ), - url( - regex=r"^site/update/(?P[0-9a-f-]+)$", + path( + "site/update/", view=views.ContainerTemplateSiteUpdateView.as_view(), name="site-update", ), - url( - regex=r"^site/delete/(?P[0-9a-f-]+)$", + path( + "site/delete/", view=views.ContainerTemplateSiteDeleteView.as_view(), name="site-delete", ), - url( - regex=r"^site/duplicate/(?P[0-9a-f-]+)$", + path( + "site/duplicate/", view=views.ContainerTemplateSiteDuplicateView.as_view(), name="site-duplicate", ), - url( - regex=r"^project/(?P[0-9a-f-]+)$", + path( + "project/", view=views.ContainerTemplateProjectListView.as_view(), name="project-list", ), - url( - regex=r"^project/detail/(?P[0-9a-f-]+)$", + path( + "project/detail/", view=views.ContainerTemplateProjectDetailView.as_view(), name="project-detail", ), - url( - regex=r"^project/create/(?P[0-9a-f-]+)$", + path( + "project/create/", view=views.ContainerTemplateProjectCreateView.as_view(), name="project-create", ), - url( - regex=r"^project/update/(?P[0-9a-f-]+)$", + path( + "project/update/", view=views.ContainerTemplateProjectUpdateView.as_view(), name="project-update", ), - url( - regex=r"^project/delete/(?P[0-9a-f-]+)$", + path( + "project/delete/", view=views.ContainerTemplateProjectDeleteView.as_view(), name="project-delete", ), - url( - regex=r"^project/duplicate/(?P[0-9a-f-]+)$", + path( + "project/duplicate/", view=views.ContainerTemplateProjectDuplicateView.as_view(), name="project-duplicate", ), - url( - regex=r"^project/copy/(?P[0-9a-f-]+)$", + path( + "project/copy/", view=views.ContainerTemplateProjectCopyView.as_view(), name="project-copy", ), # Ajax views - url( - regex=r"^ajax/get-containertemplate$", + path( + "ajax/get-containertemplate", view=views.ContainerTemplateSelectorApiView.as_view(), name="ajax-get-containertemplate", ), diff --git a/containertemplates/views.py b/containertemplates/views.py index 9b5ee10..5c57968 100644 --- a/containertemplates/views.py +++ b/containertemplates/views.py @@ -535,10 +535,10 @@ def post(self, request, *args, **kwargs): else: if data.get("containertemplatesite") is not None: try: - data[ - "containertemplatesite" - ] = ContainerTemplateSite.objects.get( - id=data.get("containertemplatesite") + data["containertemplatesite"] = ( + ContainerTemplateSite.objects.get( + id=data.get("containertemplatesite") + ) ) except ContainerTemplateSite.DoesNotExist: diff --git a/docker/Dockerfile b/docker/Dockerfile index b873f89..4dfe8e7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-buster +FROM python:3.11-buster MAINTAINER Oliver Stolpe LABEL org.opencontainers.image.source https://github.com/bihealth/kiosc-server @@ -13,7 +13,7 @@ ENV CUSTOM_STATIC_DIR /usr/src/app/local-static ENV GIT_SSH_COMMAND "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" ## Add the wait script to the image -ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.7.3/wait /usr/local/bin/wait +ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.12.1/wait /usr/local/bin/wait RUN chmod +x /usr/local/bin/wait # Copy source code into Docker image. @@ -55,12 +55,14 @@ RUN mkdir -p /usr/src/app/local-static/local/css && \ cd /usr/src/app/local-static/local/css && \ wget \ https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css \ - https://cdn.jsdelivr.net/npm/bootstrap@4.5.3/dist/css/bootstrap.min.css && \ + https://cdn.jsdelivr.net/npm/bootstrap@4.5.3/dist/css/bootstrap.min.css \ + https://cdn.jsdelivr.net/npm/bootstrap@4.5.3/dist/css/bootstrap.min.css.map && \ \ cd /usr/src/app/local-static/local/js && \ wget \ https://code.jquery.com/jquery-3.5.1.min.js \ https://cdn.jsdelivr.net/npm/bootstrap@4.5.3/dist/js/bootstrap.bundle.min.js \ + https://cdn.jsdelivr.net/npm/bootstrap@4.5.3/dist/js/bootstrap.bundle.min.js.map \ https://cdnjs.cloudflare.com/ajax/libs/tether/1.4.4/js/tether.js \ https://cdnjs.cloudflare.com/ajax/libs/shepherd/1.8.1/js/shepherd.min.js \ https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.0/clipboard.min.js diff --git a/kioscadmin/management/commands/stop_all.py b/kioscadmin/management/commands/stop_all.py index 05f5c90..2c1a983 100644 --- a/kioscadmin/management/commands/stop_all.py +++ b/kioscadmin/management/commands/stop_all.py @@ -1,4 +1,5 @@ """Django command for stopping all containers.""" + import docker.errors from django.conf import settings from django.core.management.base import BaseCommand diff --git a/kioscadmin/urls.py b/kioscadmin/urls.py index dd59ed5..c48ff01 100644 --- a/kioscadmin/urls.py +++ b/kioscadmin/urls.py @@ -1,4 +1,4 @@ -from django.conf.urls import url +from django.urls import path from . import views @@ -7,8 +7,8 @@ urlpatterns = [ - url( - regex=r"^overview$", + path( + "overview", view=views.KioscAdminView.as_view(), name="overview", ), diff --git a/requirements/base.txt b/requirements/base.txt index ec49897..00d9bb3 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,113 +1,114 @@ # Wheel -wheel>=0.40.0, <0.41 +wheel>=0.42.0, <0.43 # Setuptools -setuptools>=67.6.0, <67.7 +setuptools>=70.0.0, <70.1 # Packaging -packaging>=23.0, <24.0 +packaging>=23.2, <24.0 # Django -django>=3.2.24, <3.3 +django>=4.2.16, <5.0 # Configuration -django-environ>=0.10.0, <0.11 +django-environ>=0.11.2, <0.12 # Forms -django-crispy-forms>=2.0, <2.1 -crispy-bootstrap4==2022.1 +django-crispy-forms>=2.1, <2.2 +crispy-bootstrap4==2024.1 # Models -django-model-utils>=4.3.1, <4.4 +django-model-utils>=4.4.0, <4.5 # Password storage argon2-cffi>=21.3.0, <21.4 # Python-PostgreSQL Database Adapter -psycopg2-binary>=2.9.5, <2.10 +psycopg2-binary>=2.9.9, <2.10 # Unicode slugification awesome-slugify>=1.6.5, <1.7 # Time zones support -pytz>=2022.7.1 +pytz>=2024.1 # SVG icon support -django-iconify==0.1.1 # NOTE: v0.3 crashes, see issue +django-iconify==0.3 # NOTE: v0.3 crashes, see issue + +# OpenID Connect (OIDC) authentication support +social-auth-app-django>=5.4.0, <5.5 # Online documentation via django-docs -docutils==0.18.1 # NOTE: sphinx-rtd-theme 1.2 requires <0.19 -Sphinx==6.2.1 # NOTE: sphinx-rtd-theme v1.2.2 forces <7 +docutils==0.20.1 +Sphinx==7.2.6 django-docs==0.3.3 -sphinx-rtd-theme==1.2.2 +sphinx-rtd-theme==2.0.0 # Versioning -versioneer==0.28 +versioneer==0.29 ###################### # Project app imports ###################### # Django-plugins (with Django v3.0+ support) -django-plugins-bihealth==0.4.0 +django-plugins-bihealth==0.5.2 # Rules for permissions rules>=3.3, <3.4 # REST framework -djangorestframework>=3.14.0, <3.15 +djangorestframework>=3.15.2, <3.16 # Keyed list addon for DRF -drf-keyed-list-bihealth==0.1.1 +drf-keyed-list-bihealth==0.2.1 # Token authentication django-rest-knox>=4.2.0, <4.3 # Markdown field support -markdown==3.4.1 +markdown==3.5.2 django-markupfield>=2.0.1, <2.1 django-pagedown>=2.2.1, <2.3 -mistune>=2.0.5, <2.1 +mistune>=3.0.2, <3.1 # Database file storage for filesfolders -django-db-file-storage==0.5.5 +django-db-file-storage==0.5.6.1 # Celery dependency -redis>=4.4.4 +redis>=5.0.2 # Backround Jobs requirements -celery>=5.2.7, <5.3 +celery>=5.3.6, <5.4 # Django autocomplete light (DAL) # NOTE: 3.9.5 causes crash with Whitenoise (see issue #1224) -django-autocomplete-light==3.9.4 - -# SAML2 support for SSO -django-saml2-auth-ai>=2.1.6, <2.2 +django-autocomplete-light==3.11.0 # SODAR Core -django-sodar-core==0.13.4 +django-sodar-core==1.0.2 # Docker -docker>=6.1.0, <7.0 +docker==7.1.0 # Django reverse proxy -# django-revproxy==0.10.0 --e git+https://github.com/TracyWebTech/django-revproxy.git@9517fc26120e93e1a947f55b9dc571e68178efc0#egg=django-revproxy +django-revproxy==0.12.0 +#-e git+https://github.com/TracyWebTech/django-revproxy.git@9517fc26120e93e1a947f55b9dc571e68178efc0#egg=django-revproxy # State machine -python-statemachine==0.8.0 +python-statemachine==2.3.6 # Django Channels -channels==2.3.1 -channels_redis==2.4.2 -service_identity==18.1.0 +channels==4.1.0 +channels_redis==4.2.0 +service_identity==24.1.0 # Websockets -websockets==10.1 +websockets==13.1 +websocket-client==1.8.0 # Django redis -django-redis==5.2.0 +django-redis==5.4.0 # Requests fails with 2.32.0 -requests==2.31.0 +requests==2.32.3 diff --git a/requirements/local.txt b/requirements/local.txt index 252fd81..7940606 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -1,10 +1,15 @@ # Local development dependencies go here -r base.txt -django-extensions==3.2.1 +django-extensions==3.2.3 Werkzeug>=3.0.1, <3.1 -django-debug-toolbar>=3.8.1, <3.9 +django-debug-toolbar>=4.3.0, <4.4 # improved REPL ipdb>=0.13.13, <0.14 + +# OpenAPI support +inflection>=0.5.1, <0.6 +pyyaml>=6.0.1, <6.1 +uritemplate>=4.1.1, <4.2 diff --git a/requirements/production.txt b/requirements/production.txt index 07cafd0..9a4a758 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -4,9 +4,9 @@ # Whitenoise for static files -whitenoise==6.2.0 +whitenoise==6.7.0 # WSGI Handler -gevent==24.2.1 -daphne==2.5.0 -uvicorn==0.17.1 +gevent==24.10.3 +daphne==4.1.2 +uvicorn==0.32.0 diff --git a/requirements/test.txt b/requirements/test.txt index c05f332..bb0801c 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,30 +1,33 @@ # Test dependencies go here. -r base.txt -flake8==6.0.0 -django-test-plus==2.2.1 -factory-boy==3.2.1 +flake8==7.0.0 +django-test-plus==2.2.3 +factory-boy==3.3.0 coverage==6.5.0 -django-coverage-plugin==3.0.0 +django-coverage-plugin==3.1.0 # pytest -pytest-django==4.5.2 -pytest-sugar==0.9.6 +pytest-django==4.8.0 +pytest-sugar==1.0.0 # Selenium for UI testing -selenium==4.8.2 +selenium==4.18.1 # Tblib for tracebacks -tblib==1.7.0 +tblib==3.0.0 # BeautifulSoup for HTML testing -beautifulsoup4==4.11.2 +beautifulsoup4==4.12.3 # Black for formatting -black==23.1.0 +black==24.3.0 + +# Coveralls for coverage reporting +coveralls==3.3.1 # Coverage through Codacy codacy-coverage==1.3.11 # Mock library for urllib3 -urllib3-mock==0.3.3 +-e git+https://github.com/shipwell/urllib3-mock.git@87a2f7fb8b93fcbe6e1a82fed975eeefed247a7b#egg=urllib3_mock diff --git a/utility/install_os_dependencies.sh b/utility/install_os_dependencies.sh index a5908f5..b197bc5 100755 --- a/utility/install_os_dependencies.sh +++ b/utility/install_os_dependencies.sh @@ -32,7 +32,3 @@ echo "Installing django-extensions dependencies" echo "***********************************************" apt-get -y install graphviz-dev -echo "***********************************************" -echo "Installing SAML dependencies" -echo "***********************************************" -apt-get -y install xmlsec1