Merge branch 'dev' into dev-next-release

.github/dependabot.yml

@@ -0,0 +1,8 @@
+
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 10

ckanext-hdx_package/ckanext/hdx_package/actions/update.py

@@ -5,7 +5,6 @@
 '''
 
 import datetime
-import json
 import logging
 
 from six import text_type
@@ -31,6 +30,7 @@
 from ckanext.hdx_package.helpers.analytics import QACompletedAnalyticsSender
 from ckanext.hdx_package.helpers.constants import FILE_WAS_UPLOADED, \
     BATCH_MODE, BATCH_MODE_DONT_GROUP, BATCH_MODE_KEEP_OLD
+from ckanext.hdx_package.helpers.resource_processors.csrf_field_remover import remove_unwanted_csrf_field
 from ckanext.hdx_package.helpers.resource_triggers import \
     BEFORE_PACKAGE_UPDATE_LISTENERS, AFTER_PACKAGE_UPDATE_LISTENERS, VERSION_CHANGE_ACTIONS
 from ckanext.hdx_package.helpers.file_removal import file_remove, find_filename_in_url
@@ -225,6 +225,7 @@ def package_update(
 
     process_batch_mode(context, data_dict)
     process_skip_validation(context, data_dict)
+    remove_unwanted_csrf_field(data_dict)
 
     model = context['model']
     session = context['session']

ckanext-hdx_package/ckanext/hdx_package/helpers/resource_processors/csrf_field_remover.py

@@ -0,0 +1,17 @@
+from typing import Dict
+
+from ckan.types import Context
+
+
+def remove_unwanted_csrf_field(dataset_dict: Dict):
+    resources = dataset_dict.get('resources')
+    if resources:
+        for resource_dict in resources:
+            key = None
+            for k in resource_dict.keys():
+                if 'csrf' in k:
+                    key = k
+                    break
+            if key:
+                resource_dict.pop(key, None)
+

ckanext-hdx_package/ckanext/hdx_package/tests/conftest.py

@@ -1 +1 @@
-from ckanext.hdx_theme.tests.conftest import keep_db_tables_on_clean
+from ckanext.hdx_theme.tests.conftest import keep_db_tables_on_clean, dataset_with_uploaded_resource

ckanext-hdx_package/ckanext/hdx_package/tests/test_actions/test_resource_csrf_not_stored.py

@@ -0,0 +1,24 @@
+import pytest
+
+import ckan.plugins.toolkit as tk
+import ckan.model as model
+
+from typing import Dict, cast
+from ckan.types import Context
+
+_get_action = tk.get_action
+
+
+@pytest.mark.usefixtures('keep_db_tables_on_clean', 'clean_db', 'clean_index')
+def test_csrf_not_stored_in_resource(dataset_with_uploaded_resource: Dict):
+    resource_dict: Dict = dataset_with_uploaded_resource['resources'][0]
+    for key in resource_dict.keys():
+        assert 'csrf_token' not in key
+
+    context = cast(Context, {'model': model, 'session': model.Session, 'user': 'test_hdx_sysadmin_user'})
+    modified_resource_dict = _get_action('resource_patch')(context, {
+        'id': resource_dict['id'],
+        '_csrf_token': 'abcdef'
+    })
+    for key in modified_resource_dict.keys():
+        assert 'csrf_token' not in key, 'csrf_token should not be saved in resource'

ckanext-hdx_theme/ckanext/hdx_theme/tests/conftest.py

@@ -1,9 +1,75 @@
 import pytest
 
-from ckan import model as model
+import ckan.tests.factories as factories
+import ckan.model as model
+import ckan.plugins.toolkit as tk
 
+from typing import cast, Dict
+from collections import namedtuple
+from ckan.types import Context
+
+from ckanext.hdx_org_group.helpers.static_lists import ORGANIZATION_TYPE_LIST
+
+
+_get_action = tk.get_action
+TestInfo = namedtuple('TestInfo', ['sysadmin_id', 'user_id', 'dataset_id'])
+
+SYSADMIN_USER = 'test_hdx_sysadmin_user'
+STANDARD_USER = 'test_hdx_standard_user'
+DATASET_NAME = 'dataset_name_for_test_hdx'
+LOCATION_NAME = 'location_test_hdx'
+ORG_NAME = 'org_name_test_hdx'
 
 @pytest.fixture(scope='module')
 def keep_db_tables_on_clean():
     model.repo.tables_created_and_initialised = True
 
+
+def _get_dataset_dict() -> Dict:
+    return {
+        'package_creator': 'test function',
+        'private': False,
+        'dataset_date': '[1960-01-01 TO 2012-12-31]',
+        'caveats': 'These are the caveats',
+        'license_other': 'TEST OTHER LICENSE',
+        'methodology': 'This is a test methodology',
+        'dataset_source': 'Test data',
+        'license_id': 'hdx-other',
+        'name': DATASET_NAME,
+        'notes': 'This is a test dataset',
+        'title': 'Test Dataset ' + DATASET_NAME,
+        'owner_org': ORG_NAME,
+        'groups': [{'name': LOCATION_NAME}],
+        'data_update_frequency': '30',
+        'maintainer': STANDARD_USER
+    }
+
+
+@pytest.fixture()
+def dataset_with_uploaded_resource() -> Dict:
+    factories.User(name=STANDARD_USER, email='[email protected]')
+    factories.User(name=SYSADMIN_USER, email='[email protected]', sysadmin=True)
+    group = factories.Group(name=LOCATION_NAME)
+    factories.Organization(
+        name=ORG_NAME,
+        title='ORG NAME FOR HDX_REL_URL',
+        users=[
+            {'name': STANDARD_USER, 'capacity': 'editor'},
+        ],
+        hdx_org_type=ORGANIZATION_TYPE_LIST[0][1],
+        org_url='https://hdx.hdxtest.org/'
+    )
+    dataset_dict = _get_dataset_dict()
+    dataset_dict['resources'] = [
+        {
+            'url': 'hdx_test.csv',
+            'url_type': 'upload',
+            'resource_type': 'file.upload',
+            'format': 'CSV',
+            'name': 'hdx_test1.csv',
+            'package_id': DATASET_NAME,
+        }
+    ]
+    context = cast(Context,{'model': model, 'session': model.Session, 'user': SYSADMIN_USER})
+    created_dataset_dict = _get_action('package_create')(context, dataset_dict)
+    return created_dataset_dict