Merge branch 'dev' into feat/HDX-10138_post_CKAN_upgrade_config_decla…

…ration_yaml

ckanext-hdx_package/ckanext/hdx_package/actions/get.py

@@ -1037,7 +1037,8 @@ def hdx_recommend_tags(context, data_dict):
     tag_recommender = TagRecommender(data_dict.get('title'), data_dict.get('organization'))
     recommended_tags = tag_recommender.find_recommended_tags()
     approved_tags = get_action('cached_approved_tags_list')(context, {})
-    filtered_tags = [tag for tag in recommended_tags if tag['name'] in approved_tags]
+    filtered_tags = [tag for tag in recommended_tags if
+                     tag['name'] in approved_tags and not tag['name'].startswith('crisis-')]
     return filtered_tags
 
 

ckanext-hdx_package/ckanext/hdx_package/helpers/caching.py

@@ -3,9 +3,11 @@
 
 @author: alexandru-m-g
 '''
-
+import csv
 import logging
 import unicodedata
+from typing import Set
+
 import requests
 from dogpile.cache import make_region
 
@@ -201,3 +203,28 @@ def cached_approved_tags_list():
 def invalidate_cached_approved_tags():
     log.info('Invalidating cache for approved tags list')
     cached_approved_tags_list.invalidate()
+
+@dogpile_requests_region.cache_on_arguments()
+def cached_datasets_with_notifications() -> Set[str]:
+    log.info('Creating cache list of datasets with notifications')
+    return hdx_retrieve_datasets_with_notifications(None, None)
+
+def hdx_retrieve_datasets_with_notifications(context, data_dict) -> Set[str]:
+    url = config.get('hdx.notifications.enabled_datasets_csv')
+    if url:
+        try:
+            response = requests.get(url)
+            response.raise_for_status()  # Raise an exception for HTTP errors
+        except requests.exceptions.RequestException as e:
+            error_msg = str(e)
+            log.error(f"An error occurred: {error_msg}")
+            raise Exception(f'Couldn\'t fetch datasets with notifications from Google Spreadsheets: {error_msg}')
+
+        csv_data = response.text
+        csv_reader = csv.reader(csv_data.splitlines())
+
+        datasets = {row[0] for row in csv_reader}
+        return datasets
+    else:
+        log.error('No URL for notification-enabled datasets found in config')
+        return set()

ckanext-hdx_package/ckanext/hdx_package/helpers/custom_validator.py

@@ -738,21 +738,108 @@ def hdx_dataseries_title_validator(value, context):
     return value
 
 
-def hdx_tag_name_approved_validator(key, data, errors, context):
+def hdx_tag_name_approved_validator(key: FlattenKey, data: FlattenDataDict, errors: FlattenErrorDict,
+                                    context: Context) -> Any:
     user = context.get('user')
-    ignore_auth = context.get('ignore_auth')
-    allowed_to_add_crisis_tags = ignore_auth or (user and authz.is_sysadmin(user))
+    allowed_to_add_crisis_tags = user and authz.is_sysadmin(user)
 
     tag_name = data.get(key)
 
     approved_tags = get_action('cached_approved_tags_list')(context, {})
 
+    pkg_id = data.get(('id',))
+    if pkg_id:
+        old_tag_names = _extract_old_tag_names(context, pkg_id)
+    else:
+        old_tag_names = []
+
+    if key not in errors:
+        errors[key] = []
+
     if tag_name not in approved_tags:
         approved_tags_url = 'https://data.humdata.org/rdr/spreadsheets/approved-tags'
         errors[key].append("Tag name '{}' is not in the approved list of tags. Check the list at: {}".format(tag_name, approved_tags_url))
     # Only sysadmins are allowed to use tags starting with "crisis-"
-    if tag_name.startswith('crisis-') and not allowed_to_add_crisis_tags:
-        errors[key].append("Tag name '{}' can only be added by sysadmins".format(tag_name))
+    if tag_name.startswith('crisis-'):
+        if not allowed_to_add_crisis_tags and tag_name not in old_tag_names:
+            errors[key].append("Tag name '{}' can only be added by sysadmins".format(tag_name))
+
+
+def hdx_tag_string_approved_validator(key: FlattenKey, data: FlattenDataDict, errors: FlattenErrorDict,
+                                      context: Context) -> Any:
+    index = 0
+    while ('tags', index, 'name') in data:
+        key = ('tags', index, 'name')
+        hdx_tag_name_approved_validator(key, data, errors, context)
+        index += 1
+
+
+def hdx_keep_crisis_tag_string_if_not_sysadmin(key: FlattenKey, data: FlattenDataDict, errors: FlattenErrorDict,
+                                               context: Context) -> Any:
+    user = context.get('user')
+
+    can_remove_crisis_tags = user and authz.is_sysadmin(user)
+    has_new_tags = ('tags', 0, 'name') in data
+
+    if not can_remove_crisis_tags and not has_new_tags:
+        pkg_id = data.get(('id',))
+        if pkg_id:
+            new_tag_names_value = data.get(key)
+            if isinstance(new_tag_names_value, list):
+                new_tag_names = set(new_tag_names_value)
+            elif isinstance(new_tag_names_value, str):
+                new_tag_names = set(new_tag_names_value.split(','))
+            else:
+                new_tag_names = set()
+            old_tag_names = _extract_old_tag_names(context, pkg_id)
+
+            crisis_tags = [tag for tag in old_tag_names if tag.startswith('crisis-') and tag not in new_tag_names]
+
+            combined_tags = new_tag_names.union(crisis_tags)
+            data[key] = ','.join(combined_tags)
+
+
+def hdx_keep_crisis_tags_if_not_sysadmin(key: FlattenKey, data: FlattenDataDict, errors: FlattenErrorDict,
+                                         context: Context) -> Any:
+    user = context.get('user')
+
+    can_remove_crisis_tags = user and authz.is_sysadmin(user)
+
+    if not can_remove_crisis_tags:
+        pkg_id = data.get(('id',))
+        if pkg_id:
+            current_tags = _extract_tag_names(data)
+            old_tag_names = _extract_old_tag_names(context, pkg_id)
+
+            i = len(current_tags)
+            for old_tag in old_tag_names:
+                if old_tag.startswith('crisis-') and old_tag not in current_tags:
+                    data[('tags', i, 'name')] = old_tag
+                    current_tags.append(old_tag)
+                    i += 1
+
+
+def _extract_tag_names(data: FlattenDataDict) -> list:
+    tag_names = []
+
+    index = 0
+    while ('tags', index, 'name') in data:
+        tag_names.append(data.get(('tags', index, 'name'), ''))
+        index += 1
+
+    return tag_names
+
+
+def _extract_old_tag_names(context: Context, pkg_id: str) -> list:
+    tag_names = []
+
+    prev_package_dict = __get_previous_package_dict(context, pkg_id)
+    tags = prev_package_dict.get('tags', [])
+
+    for tag in tags:
+        tag_names.append(tag.get('display_name'))
+
+    return tag_names
 
 
 def hdx_update_last_modified_if_url_changed(key: FlattenKey, data: FlattenDataDict,

ckanext-hdx_package/ckanext/hdx_package/plugin.py

@@ -200,12 +200,19 @@ def _modify_package_schema(self, schema):
                 tk.get_validator('ignore_missing'),
                 tk.get_validator('hdx_dataseries_title_validator'),
                 tk.get_converter('convert_to_extras')
+            ],
+            'tag_string': [
+                tk.get_validator('hdx_keep_crisis_tag_string_if_not_sysadmin'),
+                tk.get_validator('ignore_missing'),
+                tk.get_validator('tag_string_convert'),
+                tk.get_validator('hdx_tag_string_approved_validator'),
             ]
         })
 
         schema['tags'].update(
             {
                 'name': [
+                    tk.get_validator('hdx_keep_crisis_tags_if_not_sysadmin'),
                     tk.get_validator('not_missing'),
                     tk.get_validator('not_empty'),
                     tk.get_validator('unicode_safe'),
@@ -537,6 +544,9 @@ def get_validators(self):
             'hdx_keep_if_fs_check_format': vd.hdx_keep_if_fs_check_format,
             'hdx_add_update_fs_check_info': vd.hdx_add_update_fs_check_info,
             'hdx_tag_name_approved_validator': vd.hdx_tag_name_approved_validator,
+            'hdx_tag_string_approved_validator': vd.hdx_tag_string_approved_validator,
+            'hdx_keep_crisis_tag_string_if_not_sysadmin': vd.hdx_keep_crisis_tag_string_if_not_sysadmin,
+            'hdx_keep_crisis_tags_if_not_sysadmin': vd.hdx_keep_crisis_tags_if_not_sysadmin,
             'hdx_update_last_modified_if_url_changed': vd.hdx_update_last_modified_if_url_changed,
             'hdx_disable_live_frequency_filestore_resources_only': vd.hdx_disable_live_frequency_filestore_resources_only,
             'hdx_in_hapi_flag_values': vd.hdx_value_in_list_wrapper(IN_HAPI_FLAG_VALUES, True),

ckanext-hdx_package/ckanext/hdx_package/tests/test_actions/test_package_update.py

@@ -423,6 +423,30 @@ def test_hdx_package_tags_validation(self):
             assert "Tag name '{}' can only be added by sysadmins".format(crisis_tag_name) in e.error_dict.get('tags')[
                 0], 'Only sysadmins are allowed to add tags starting with "crisis-"'
 
+        data_dict = self._modify_field(context_user, joeadmin, package['name'], 'tags',
+                                       [{'name': crisis_tag_name}, {'name': 'disease'}])
+        modified_package = data_dict.get('modified_package')
+
+        assert len(modified_package.get('tags')) == 2
+        assert crisis_tag_name in [tag['name'] for tag in modified_package.get(
+            'tags')], 'Crisis tags should be kept if specified by a user, as they were already added by a sysadmin'
+        assert 'disease' in [tag['name'] for tag in modified_package.get('tags')]
+
+        data_dict = self._modify_field(context_user, joeadmin, package['name'], 'tags', [{'name': 'boys'}])
+        modified_package = data_dict.get('modified_package')
+
+        assert len(modified_package.get('tags')) == 2
+        assert 'boys' in [tag['name'] for tag in modified_package.get('tags')], \
+            'Crisis tags should be kept even if not specified by a user, as they were already added by a sysadmin'
+
+        data_dict = self._modify_field(context, testsysadmin, package['name'], 'tags',
+                                       [{'name': 'boys'}, {'name': 'disease'}])
+        modified_package = data_dict.get('modified_package')
+
+        assert len(modified_package.get('tags')) == 2
+        assert 'boys' in [tag['name'] for tag in modified_package.get('tags')], \
+            'Crisis tags should be removed if not specified by a sysadmin'
+
 
     def _modify_field(self, context, user, package_id, key, value):
         modified_fields = {'id': package_id,

ckanext-hdx_package/ckanext/hdx_package/tests/test_controller/test_contribute_flow.py

@@ -241,7 +241,7 @@ def test_edit_lists(self):
         auth = {'Authorization': self.testsysadmin_token}
 
         post_params = self._get_dataset_post_param('testing-dataset-edit-lists')
-        post_params['tag_string'] = 'list_test_tag'
+        post_params['tag_string'] = 'boys'
 
         post_params["dataset_date"] = "[1960-01-01 TO 2012-12-31]"
         post_params['save'] = 'update-dataset-json'

ckanext-hdx_package/ckanext/hdx_package/views/dataset.py

@@ -26,6 +26,7 @@
 import ckanext.hdx_package.controller_logic.dataset_view_logic as dataset_view_logic
 from ckanext.hdx_package.controller_logic.dataset_contact_contributor_logic import DatasetContactContributorLogic
 from ckanext.hdx_package.controller_logic.dataset_request_access_logic import DatasetRequestAccessLogic
+from ckanext.hdx_users.controller_logic.notification_platform_logic import verify_unsubscribe_token
 
 from ckan.views.dataset import _setup_template_variables
 
@@ -40,6 +41,7 @@
 from ckanext.hdx_theme.util.light_redirect import check_redirect_needed
 
 from ckanext.hdx_org_group.views.organization_join import set_custom_rect_logo_url
+from ckanext.hdx_users.helpers.notification_platform import check_notifications_enabled_for_dataset
 
 log = logging.getLogger(__name__)
 
@@ -182,13 +184,24 @@ def read(id):
     else:
         logo_config = {}
 
+    # notification platform
+    supports_notifications = check_notifications_enabled_for_dataset(pkg_dict['id'])
+    unsubscribe_token = request.args.get('unsubscribe_token', None)
+    if unsubscribe_token:
+        try:
+            token_obj = verify_unsubscribe_token(unsubscribe_token, inactivate=False)
+        except Exception as e:
+            unsubscribe_token = None
+            h.flash_error('Your token is invalid or has expired.')
+
     template_data = {
         'pkg_dict': pkg_dict,
         'pkg': pkg,
         'showcase_list': showcase_list,
         'hdx_activities': hdx_activities,
         'membership': membership,
         'user_has_edit_rights': user_has_edit_rights,
+        'unsubscribe_token': unsubscribe_token,
         'analytics_is_cod': analytics_is_cod,
         'analytics_is_indicator': 'false',
         'analytics_is_archived': analytics_is_archived,
@@ -198,6 +211,7 @@ def read(id):
         'stats_downloads_last_weeks': stats_downloads_last_weeks,
         'user_survey_url': user_survey_url,
         'logo_config': logo_config,
+        'supports_notifications': supports_notifications,
     }
 
     if _dataset_preview != vd._DATASET_PREVIEW_NO_PREVIEW:

ckanext-hdx_package/ckanext/hdx_package/views/light_dataset.py

@@ -11,6 +11,8 @@
 from ckanext.hdx_search.controller_logic.search_logic import SearchLogic, ArchivedUrlHelper
 from ckanext.hdx_theme.util.http_exception_helper import catch_http_exceptions
 from ckanext.hdx_theme.util.light_redirect import check_redirect_needed
+from ckanext.hdx_users.controller_logic.notification_platform_logic import verify_unsubscribe_token
+from ckanext.hdx_users.helpers.notification_platform import check_notifications_enabled_for_dataset
 
 get_action = tk.get_action
 check_access = tk.check_access
@@ -68,10 +70,22 @@ def read(id):
         dataset_dict['page_list'] = cp_h.hdx_get_page_list_for_dataset(context, dataset_dict)
         dataset_dict['link_list'] = get_action('hdx_package_links_by_id_list')(context, {'id': dataset_dict.get('name')})
 
+        # notification platform
+        supports_notifications = check_notifications_enabled_for_dataset(dataset_dict['id'])
+        unsubscribe_token = tk.request.args.get('unsubscribe_token', None)
+        if unsubscribe_token:
+            try:
+                token_obj = verify_unsubscribe_token(unsubscribe_token, inactivate=False)
+            except Exception as e:
+                unsubscribe_token = None
+                tk.h.flash_error('Your token is invalid or has expired.')
+
         template_data = {
             'dataset_dict': dataset_dict,
             'analytics': analytics_dict,
-            'user_survey_url': user_survey_url
+            'user_survey_url': user_survey_url,
+            'unsubscribe_token': unsubscribe_token,
+            'supports_notifications': supports_notifications,
         }
 
         return render(u'light/dataset/read.html', template_data)