From 116606c422fddcf5d521cd2c424a5477158d60bd Mon Sep 17 00:00:00 2001 From: Serban Teodorescu Date: Wed, 21 Aug 2024 15:08:21 +0300 Subject: [PATCH 1/3] Add Dependabot configuration for pip --- .github/dependabot.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..4da583b382 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ + +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 10 From bbdcfe9be5d8c87d509e3183ade8a5f65adec544 Mon Sep 17 00:00:00 2001 From: alexandru-m-g Date: Thu, 22 Aug 2024 01:41:29 +0300 Subject: [PATCH 2/3] HDX-10108 remove any csrf fields from resource dict before saving to db --- .../ckanext/hdx_package/actions/update.py | 3 +- .../resource_processors/csrf_field_remover.py | 17 +++++ .../ckanext/hdx_package/tests/conftest.py | 2 +- .../test_resource_csrf_not_stored.py | 24 +++++++ .../ckanext/hdx_theme/tests/conftest.py | 68 ++++++++++++++++++- 5 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 ckanext-hdx_package/ckanext/hdx_package/helpers/resource_processors/csrf_field_remover.py create mode 100644 ckanext-hdx_package/ckanext/hdx_package/tests/test_actions/test_resource_csrf_not_stored.py diff --git a/ckanext-hdx_package/ckanext/hdx_package/actions/update.py b/ckanext-hdx_package/ckanext/hdx_package/actions/update.py index b01f7a6b5b..4faee808bc 100644 --- a/ckanext-hdx_package/ckanext/hdx_package/actions/update.py +++ b/ckanext-hdx_package/ckanext/hdx_package/actions/update.py @@ -5,7 +5,6 @@ ''' import datetime -import json import logging from six import text_type @@ -31,6 +30,7 @@ from ckanext.hdx_package.helpers.analytics import QACompletedAnalyticsSender from ckanext.hdx_package.helpers.constants import FILE_WAS_UPLOADED, \ BATCH_MODE, BATCH_MODE_DONT_GROUP, BATCH_MODE_KEEP_OLD +from ckanext.hdx_package.helpers.resource_processors.csrf_field_remover import remove_unwanted_csrf_field from ckanext.hdx_package.helpers.resource_triggers import \ BEFORE_PACKAGE_UPDATE_LISTENERS, AFTER_PACKAGE_UPDATE_LISTENERS, VERSION_CHANGE_ACTIONS from ckanext.hdx_package.helpers.file_removal import file_remove, find_filename_in_url @@ -225,6 +225,7 @@ def package_update( process_batch_mode(context, data_dict) process_skip_validation(context, data_dict) + remove_unwanted_csrf_field(data_dict) model = context['model'] session = context['session'] diff --git a/ckanext-hdx_package/ckanext/hdx_package/helpers/resource_processors/csrf_field_remover.py b/ckanext-hdx_package/ckanext/hdx_package/helpers/resource_processors/csrf_field_remover.py new file mode 100644 index 0000000000..58524114c8 --- /dev/null +++ b/ckanext-hdx_package/ckanext/hdx_package/helpers/resource_processors/csrf_field_remover.py @@ -0,0 +1,17 @@ +from typing import Dict + +from ckan.types import Context + + +def remove_unwanted_csrf_field(dataset_dict: Dict): + resources = dataset_dict.get('resources') + if resources: + for resource_dict in resources: + key = None + for k in resource_dict.keys(): + if 'csrf' in k: + key = k + break + if key: + resource_dict.pop(key, None) + diff --git a/ckanext-hdx_package/ckanext/hdx_package/tests/conftest.py b/ckanext-hdx_package/ckanext/hdx_package/tests/conftest.py index e2ee852355..6fc6665719 100644 --- a/ckanext-hdx_package/ckanext/hdx_package/tests/conftest.py +++ b/ckanext-hdx_package/ckanext/hdx_package/tests/conftest.py @@ -1 +1 @@ -from ckanext.hdx_theme.tests.conftest import keep_db_tables_on_clean +from ckanext.hdx_theme.tests.conftest import keep_db_tables_on_clean, dataset_with_uploaded_resource diff --git a/ckanext-hdx_package/ckanext/hdx_package/tests/test_actions/test_resource_csrf_not_stored.py b/ckanext-hdx_package/ckanext/hdx_package/tests/test_actions/test_resource_csrf_not_stored.py new file mode 100644 index 0000000000..506f44a9e8 --- /dev/null +++ b/ckanext-hdx_package/ckanext/hdx_package/tests/test_actions/test_resource_csrf_not_stored.py @@ -0,0 +1,24 @@ +import pytest + +import ckan.plugins.toolkit as tk +import ckan.model as model + +from typing import Dict, cast +from ckan.types import Context + +_get_action = tk.get_action + + +@pytest.mark.usefixtures('keep_db_tables_on_clean', 'clean_db', 'clean_index') +def test_csrf_not_stored_in_resource(dataset_with_uploaded_resource: Dict): + resource_dict: Dict = dataset_with_uploaded_resource['resources'][0] + for key in resource_dict.keys(): + assert 'csrf_token' not in key + + context = cast(Context, {'model': model, 'session': model.Session, 'user': 'test_hdx_sysadmin_user'}) + modified_resource_dict = _get_action('resource_patch')(context, { + 'id': resource_dict['id'], + '_csrf_token': 'abcdef' + }) + for key in modified_resource_dict.keys(): + assert 'csrf_token' not in key, 'csrf_token should not be saved in resource' diff --git a/ckanext-hdx_theme/ckanext/hdx_theme/tests/conftest.py b/ckanext-hdx_theme/ckanext/hdx_theme/tests/conftest.py index 3fef26e58a..cb95cfa13e 100644 --- a/ckanext-hdx_theme/ckanext/hdx_theme/tests/conftest.py +++ b/ckanext-hdx_theme/ckanext/hdx_theme/tests/conftest.py @@ -1,9 +1,75 @@ import pytest -from ckan import model as model +import ckan.tests.factories as factories +import ckan.model as model +import ckan.plugins.toolkit as tk +from typing import cast, Dict +from collections import namedtuple +from ckan.types import Context + +from ckanext.hdx_org_group.helpers.static_lists import ORGANIZATION_TYPE_LIST + + +_get_action = tk.get_action +TestInfo = namedtuple('TestInfo', ['sysadmin_id', 'user_id', 'dataset_id']) + +SYSADMIN_USER = 'test_hdx_sysadmin_user' +STANDARD_USER = 'test_hdx_standard_user' +DATASET_NAME = 'dataset_name_for_test_hdx' +LOCATION_NAME = 'location_test_hdx' +ORG_NAME = 'org_name_test_hdx' @pytest.fixture(scope='module') def keep_db_tables_on_clean(): model.repo.tables_created_and_initialised = True + +def _get_dataset_dict() -> Dict: + return { + 'package_creator': 'test function', + 'private': False, + 'dataset_date': '[1960-01-01 TO 2012-12-31]', + 'caveats': 'These are the caveats', + 'license_other': 'TEST OTHER LICENSE', + 'methodology': 'This is a test methodology', + 'dataset_source': 'Test data', + 'license_id': 'hdx-other', + 'name': DATASET_NAME, + 'notes': 'This is a test dataset', + 'title': 'Test Dataset ' + DATASET_NAME, + 'owner_org': ORG_NAME, + 'groups': [{'name': LOCATION_NAME}], + 'data_update_frequency': '30', + 'maintainer': STANDARD_USER + } + + +@pytest.fixture() +def dataset_with_uploaded_resource() -> Dict: + factories.User(name=STANDARD_USER, email='test_hdx_standard_user@hdx.hdxtest.org') + factories.User(name=SYSADMIN_USER, email='test_hdx_sysadmin_user@hdx.hdxtest.org', sysadmin=True) + group = factories.Group(name=LOCATION_NAME) + factories.Organization( + name=ORG_NAME, + title='ORG NAME FOR HDX_REL_URL', + users=[ + {'name': STANDARD_USER, 'capacity': 'editor'}, + ], + hdx_org_type=ORGANIZATION_TYPE_LIST[0][1], + org_url='https://hdx.hdxtest.org/' + ) + dataset_dict = _get_dataset_dict() + dataset_dict['resources'] = [ + { + 'url': 'hdx_test.csv', + 'url_type': 'upload', + 'resource_type': 'file.upload', + 'format': 'CSV', + 'name': 'hdx_test1.csv', + 'package_id': DATASET_NAME, + } + ] + context = cast(Context,{'model': model, 'session': model.Session, 'user': SYSADMIN_USER}) + created_dataset_dict = _get_action('package_create')(context, dataset_dict) + return created_dataset_dict From f59ef0335723e88bd9b50c2f330e0469705d9ee5 Mon Sep 17 00:00:00 2001 From: Dan Mihaila Date: Thu, 22 Aug 2024 13:48:22 +0300 Subject: [PATCH 3/3] Update version.py --- ckanext-hdx_theme/ckanext/hdx_theme/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext-hdx_theme/ckanext/hdx_theme/version.py b/ckanext-hdx_theme/ckanext/hdx_theme/version.py index 41885f7659..a7d603acf3 100644 --- a/ckanext-hdx_theme/ckanext/hdx_theme/version.py +++ b/ckanext-hdx_theme/ckanext/hdx_theme/version.py @@ -1 +1 @@ -hdx_version = 'v1.82.11' +hdx_version = 'v1.82.12'