Merge pull request #6441 from OCHA-DAP/feat/HDX-10138_post_CKAN_upgra…

…de_config_declaration_yaml

(Draft) HDX-10138 - Post CKAN upgrade - conf yaml

ckan/lib/search/__init__.py

@@ -205,7 +205,7 @@ def rebuild(package_id: Optional[str] = None,
     })
 
     from ckan.common import config, asbool
-    use_hdx_reindex = asbool(config.get('hdx.reindexing.enabled', True))
+    use_hdx_reindex = config.get('hdx.reindexing.enabled')
     if use_hdx_reindex:
         from ckanext.hdx_search.helpers.reindexing import HdxSolrReindexer
         hdx_reindexer = HdxSolrReindexer(context, model, package_index, query_for, text_traceback)

ckanext-hdx_dataviz/ckanext/hdx_dataviz/config_declaration.yaml

@@ -0,0 +1,8 @@
+version: 1
+groups:
+  - annotation: Ignore Plugin
+    options:
+      - key: hdx.ignore.plugin.hdx_dataviz
+###############################################################################
+#  All options are in the config_declarations.yml of the hdx_theme extension  
+###############################################################################

ckanext-hdx_dataviz/ckanext/hdx_dataviz/plugin.py

@@ -12,7 +12,7 @@
 
 request = toolkit.request
 
-
+@toolkit.blanket.config_declarations
 class HdxDatavizPlugin(plugins.SingletonPlugin):
     plugins.implements(plugins.IConfigurer)
     # if dataviz:

ckanext-hdx_hxl_preview/ckanext/hdx_hxl_preview/config_declaration.yaml

@@ -0,0 +1,8 @@
+version: 1
+groups:
+  - annotation: Ignore Plugin
+    options:
+      - key: hdx.ignore.plugin.hdx_hxl_preview
+###############################################################################
+#  All options are in the config_declarations.yml of the hdx_theme extension  
+###############################################################################

ckanext-hdx_hxl_preview/ckanext/hdx_hxl_preview/plugin.py

@@ -18,6 +18,7 @@
 
 ignore_empty = plugins.toolkit.get_validator('ignore_empty')
 
+@toolkit.blanket.config_declarations
 class HdxHxlPreviewPlugin(plugins.SingletonPlugin):
     plugins.implements(plugins.IConfigurer)
     plugins.implements(plugins.IResourceView, inherit=True)

ckanext-hdx_office_preview/ckanext/hdx_office_preview/config_declaration.yaml

@@ -0,0 +1,8 @@
+version: 1
+groups:
+  - annotation: Ignore Plugin
+    options:
+      - key: hdx.ignore.plugin.hdx_office_preview
+###############################################################################
+#  All options are in the config_declarations.yml of the hdx_theme extension  
+###############################################################################

ckanext-hdx_office_preview/ckanext/hdx_office_preview/plugin.py

@@ -4,6 +4,8 @@
 from ckan.common import config
 
 log = logging.getLogger(__name__)
+
+@toolkit.blanket.config_declarations
 class Hdx_Office_PreviewPlugin(plugins.SingletonPlugin):
     plugins.implements(plugins.IConfigurer)
     plugins.implements(plugins.IResourceView, inherit=True)

ckanext-hdx_org_group/ckanext/hdx_org_group/actions/create.py

@@ -87,7 +87,7 @@ def hdx_user_invite(context, data_dict):
         _get_action('group_member_create')(context, member_dict)
         group_dict = _get_action('group_show')({}, {'id': data['group_id']})
     try:
-        expiration_in_hours = int(config.get('hdx.password.invitation_reset_key.expiration_in_hours', 48))
+        expiration_in_hours = config.get('hdx.password.invitation_reset_key.expiration_in_hours')
         reset_password.create_reset_key(user, expiration_in_minutes=60 * expiration_in_hours)
         subject = u'HDX account creation'
         email_data = {

ckanext-hdx_org_group/ckanext/hdx_org_group/config_declaration.yaml

@@ -0,0 +1,8 @@
+version: 1
+groups:
+  - annotation: Ignore Plugin
+    options:
+      - key: hdx.ignore.plugin.hdx_org_group
+###############################################################################
+#  All options are in the config_declarations.yml of the hdx_theme extension  
+###############################################################################

ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/caching.py

@@ -25,7 +25,7 @@
 def cached_data_completeness(location_code):
     log.info('Fetching data completness for ' + location_code)
     url_pattern = config.get('hdx.datagrid.config_url_pattern')
-    for_prod = config.get('hdx.datagrid.prod') == 'true'
+    for_prod = config.get('hdx.datagrid.prod')
     branch = 'master' if for_prod else location_code
     url = url_pattern.format(branch=branch, iso=location_code)
 

ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/country_helper.py

@@ -254,7 +254,7 @@ def _db_to_form_schema(group_type=None):
 
 
 def _get_data_completeness(location_code):
-    cached = config.get('hdx.datagrid.prod') == 'true'
+    cached = config.get('hdx.datagrid.prod')
     data = None
     if cached:
         data = caching.cached_data_completeness(location_code)

ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/org_batch.py

@@ -40,8 +40,7 @@ def get_batch(org_identifier):
 
     org_specific_period_config = '{}_{}'.format(BATCH_PERIOD_CONFIG, group.name)
 
-    max_minutes = int(config.get(org_specific_period_config,
-                                 config.get(BATCH_PERIOD_CONFIG, '3')))
+    max_minutes = config.get(org_specific_period_config, config.get(BATCH_PERIOD_CONFIG))
 
     organization_id = group.id
 

ckanext-hdx_org_group/ckanext/hdx_org_group/plugin.py

@@ -17,7 +17,7 @@
 
 log = logging.getLogger(__name__)
 
-
+@tk.blanket.config_declarations
 class HDXOrgGroupPlugin(plugins.SingletonPlugin, lib_plugins.DefaultOrganizationForm):
     plugins.implements(plugins.IAuthFunctions)
     plugins.implements(plugins.IConfigurer, inherit=False)

ckanext-hdx_package/ckanext/hdx_package/actions/get.py

@@ -537,16 +537,16 @@ def _additional_hdx_resource_show_processing(context, resource_dict, just_for_re
     if not resource_dict.get('last_modified'):
         resource_dict['last_modified'] = resource_dict['metadata_modified']
 
-    if config.get('hdx.apihighways.enabled') == 'true':
-        resource_dict['apihighways_id'] = _get_resource_id_apihighways(resource_dict.get('id'))
-        if resource_dict['apihighways_id']:
-            resource_dict['apihighways_url'] = config.get('hdx.apihighways.baseurl') + resource_dict.get(
-                'apihighways_id')
-    else:
-        if 'apihighways_id' in resource_dict:
-            del resource_dict['apihighways_id']
-        if 'apihighways_url' in resource_dict:
-            del resource_dict['apihighways_url']
+    # if config.get('hdx.apihighways.enabled'):
+    #     resource_dict['apihighways_id'] = _get_resource_id_apihighways(resource_dict.get('id'))
+    #     if resource_dict['apihighways_id']:
+    #         resource_dict['apihighways_url'] = config.get('hdx.apihighways.baseurl') + resource_dict.get(
+    #             'apihighways_id')
+    # else:
+    #     if 'apihighways_id' in resource_dict:
+    #         del resource_dict['apihighways_id']
+    #     if 'apihighways_url' in resource_dict:
+    #         del resource_dict['apihighways_url']
     if resource_dict.get('url'):
         _process_url(context, resource_dict)
     if not just_for_reindexing:
@@ -1109,7 +1109,7 @@ def hdx_guess_format_from_extension(context, data_dict):
 def hdx_send_mail_request_tags(context, data_dict):
     _check_access('hdx_send_mail_request_tags', context, data_dict)
 
-    hdx_email = config.get('hdx.faqrequest.email', '[email protected]')
+    hdx_email = config.get('hdx.faqrequest.email')
 
     subject = u'New tag(s) request'
     email_data = {

ckanext-hdx_package/ckanext/hdx_package/config_declaration.yaml

@@ -0,0 +1,8 @@
+version: 1
+groups:
+  - annotation: Ignore Plugin
+    options:
+      - key: hdx.ignore.plugin.hdx_package
+###############################################################################
+#  All options are in the config_declarations.yml of the hdx_theme extension  
+###############################################################################

ckanext-hdx_package/ckanext/hdx_package/controller_logic/dataset_contact_contributor_logic.py

@@ -56,7 +56,7 @@ def send_mail(self):
             'pkg_title': self.request.form.get('pkg_title'),
             'pkg_id': self.request.form.get('pkg_id'),
             'pkg_url': h.url_for('dataset_read', id=self.request.form.get('pkg_id'), qualified=True),
-            'hdx_email': config.get('hdx.faqrequest.email', '[email protected]'),
+            'hdx_email': config.get('hdx.faqrequest.email'),
         }
 
         get_action('hdx_send_mail_contributor')(self.context, data_dict)

ckanext-hdx_package/ckanext/hdx_package/helpers/caching.py

@@ -178,13 +178,13 @@ def cached_resource_id_apihighways():
     log.info("Creating cache for HDX resource_id to apihighways dataset_id mapping")
 
     result = {}
-    if config.get('hdx.apihighways.enabled') == 'true':
-        try:
-            response = requests.get(config.get('hdx.apihighways.url'))
-            response.raise_for_status()
-            result = response.json()
-        except Exception as ex:
-            log.error(ex)
+    # if config.get('hdx.apihighways.enabled') == 'true':
+    #     try:
+    #         response = requests.get(config.get('hdx.apihighways.url'))
+    #         response.raise_for_status()
+    #         result = response.json()
+    #     except Exception as ex:
+    #         log.error(ex)
     return result
 
 

ckanext-hdx_package/ckanext/hdx_package/helpers/helpers.py

@@ -454,7 +454,7 @@ def generate_mandatory_fields():
         'dataset_source': selected_org.get('title'),
         'maintainer': user,
         'subnational': 1,
-        'data_update_frequency': config.get('hdx.default_frequency', '-999'),
+        'data_update_frequency': config.get('hdx.default_frequency'),
         'dataset_preview_check': '1',
         'dataset_preview': vd._DATASET_PREVIEW_FIRST_RESOURCE,
         'dataset_preview_value': vd._DATASET_PREVIEW_FIRST_RESOURCE

ckanext-hdx_package/ckanext/hdx_package/helpers/p_code_filters_helper.py

@@ -4,5 +4,5 @@
 
 
 def are_new_p_code_filters_enabled():
-    new_p_code_filters_enabled = config.get('hdx.p_code.new_filters.enabled') == 'true'
+    new_p_code_filters_enabled = config.get('hdx.p_code.new_filters.enabled')
     return new_p_code_filters_enabled

ckanext-hdx_package/ckanext/hdx_package/plugin.py

@@ -45,8 +45,8 @@
 
 
 def run_on_startup():
-    cache_on_startup = config.get('hdx.cache.onstartup', 'true')
-    if 'true' == cache_on_startup:
+    cache_on_startup = config.get('hdx.cache.onstartup')
+    if cache_on_startup:
         _generate_license_list()
         # caching.cached_get_group_package_stuff()
 
@@ -83,7 +83,7 @@ def _generate_license_list():
 def cached_group_list():
     return tk.get_action('cached_group_list')()
 
-
+@tk.blanket.config_declarations
 class HDXPackagePlugin(plugins.SingletonPlugin, tk.DefaultDatasetForm):
     plugins.implements(plugins.IAuthFunctions)
     plugins.implements(plugins.IConfigurer, inherit=False)
@@ -598,7 +598,8 @@ def validate(self, context, data_dict, schema, action):
             if is_requestdata_type:
                 self._update_with_requestdata_modify_package_schema(schema)
 
-            fields_to_skip = config.get('hdx.validation.allow_skip_for_sysadmin', '').split(',')
+            allow_skip_for_sysadmin = config.get('hdx.validation.allow_skip_for_sysadmin') or ''
+            fields_to_skip = allow_skip_for_sysadmin.split(',')
             if len(fields_to_skip) > 0 and fields_to_skip[0] and \
                     self._user_allowed_to_skip_validation(c.user) and context.get(hdx_update.SKIP_VALIDATION):
                 self._update_with_skip_validation(schema, fields_to_skip)

ckanext-hdx_package/ckanext/hdx_package/views/contact.py

@@ -75,7 +75,7 @@ def contact_members():
             data_dict['pkg_id'] = request.form.get('pkg_id')
             data_dict['pkg_url'] = h.url_for('dataset_read', id=request.form.get('pkg_id'),
                                              qualified=True)
-        data_dict['hdx_email'] = config.get('hdx.faqrequest.email', '[email protected]')
+        data_dict['hdx_email'] = config.get('hdx.faqrequest.email')
 
         hdx_validate_email(data_dict['email'])
 

ckanext-hdx_pages/ckanext/hdx_pages/config_declaration.yaml

@@ -0,0 +1,8 @@
+version: 1
+groups:
+  - annotation: Ignore Plugin
+    options:
+      - key: hdx.ignore.plugin.hdx_pages
+###############################################################################
+#  All options are in the config_declarations.yml of the hdx_theme extension  
+###############################################################################

ckanext-hdx_pages/ckanext/hdx_pages/plugin.py

@@ -12,6 +12,7 @@
 
 import ckanext.hdx_pages.views.light_page as light_page
 
+@toolkit.blanket.config_declarations
 class HdxCustomPagesPlugin(plugins.SingletonPlugin):
     plugins.implements(plugins.IBlueprint)
     plugins.implements(plugins.IActions, inherit=True)

ckanext-hdx_search/ckanext/hdx_search/config_declaration.yaml

@@ -0,0 +1,8 @@
+version: 1
+groups:
+  - annotation: Ignore Plugin
+    options:
+      - key: hdx.ignore.plugin.hdx_search
+###############################################################################
+#  All options are in the config_declarations.yml of the hdx_theme extension  
+###############################################################################

ckanext-hdx_search/ckanext/hdx_search/helpers/reindexing.py

@@ -74,7 +74,7 @@ def rebuild(self,
 
     def _hdx_fast_reindex(self, context, package_ids, package_index, defer_commit, force, quiet):
         total_packages = len(package_ids)
-        step = int(config.get('hdx.reindexing.batch_size', '100'))
+        step = config.get('hdx.reindexing.batch_size')
         start = 0
         stop = step
         counter = 0

ckanext-hdx_search/ckanext/hdx_search/model.py

@@ -48,7 +48,7 @@ def latest_queries_for_user(cls, user_id):
         :rtype: list[SearchedString]
         '''
 
-        hours_from_actual_search = asint(config.get('hdx.search_history.hours_from_actual_search', '24'))
+        hours_from_actual_search = config.get('hdx.search_history.hours_from_actual_search')
 
         hours_ago = datetime.datetime.utcnow() - datetime.timedelta(hours=hours_from_actual_search)
         query = meta.Session.query(cls).filter(cls.user == user_id).filter(cls.last_modified < hours_ago)\

ckanext-hdx_search/ckanext/hdx_search/plugin.py

@@ -41,7 +41,7 @@ def convert_country(q):
             q += ' '+c['name']
     return q
 
-
+@tk.blanket.config_declarations
 class HDXSearchPlugin(plugins.SingletonPlugin):
     plugins.implements(plugins.IConfigurer, inherit=False)
     # plugins.implements(plugins.IRoutes, inherit=True)

ckanext-hdx_search/ckanext/hdx_search/tests/test_qa_controller.py

@@ -59,10 +59,10 @@ def test_qa_dashboard(self):
         qa_dashboard_result = self._get_url(url, token)
         assert qa_dashboard_result.status_code == 200
 
-        config['hdx.qadashboard.enabled'] = 'false'
+        config['hdx.qadashboard.enabled'] = False
         result = self._get_url(url, token)
         assert result.status_code == 404
 
-        config['hdx.qadashboard.enabled'] = 'true'
+        config['hdx.qadashboard.enabled'] = True
         result = self._get_url(url, token)
         assert result.status_code == 200

ckanext-hdx_search/ckanext/hdx_search/views/qa.py

@@ -27,8 +27,8 @@
 
 
 def dashboard():
-    qa_dashboard_enabled = config.get('hdx.qadashboard.enabled', 'true')
-    if qa_dashboard_enabled == 'false':
+    qa_dashboard_enabled = config.get('hdx.qadashboard.enabled')
+    if not qa_dashboard_enabled:
         abort(404, _('Page not found'))
 
     try:

ckanext-hdx_service_checker/ckanext/hdx_service_checker/actions/authorize.py

@@ -10,6 +10,6 @@ def run_checks(context, data_dict=None):
     '''
     Only sysadmins are allowed to call this action
     '''
-    if request.user_agent.string == config.get('hdx.checks.allowed_user_agent', 'HDX-CHECKER'):
+    if request.user_agent.string == config.get('hdx.checks.allowed_user_agent'):
         return {'success': True}
     return {'success': False, 'msg': _('Only sysadmins can run checks')}

ckanext-hdx_service_checker/ckanext/hdx_service_checker/checker.py

@@ -96,7 +96,7 @@ def run_checks(self):
     logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s: %(message)s',
                         datefmt='%m/%d/%Y %I:%M:%S %p', handlers=[logging.StreamHandler()])
     runtime_vars = {
-        'hdx.rest.indicator.endpoint': 'https://manage.humdata.org/public/api2/values',
+    #    'hdx.rest.indicator.endpoint': 'https://manage.humdata.org/public/api2/values',
         'SOLR_URL': 'http://172.17.42.1:9011/solr/ckan/select?q=health&start=0&rows=1'
     }
     config_file = '/home/alex/PycharmProjects/hdx-ckan/ckanext-hdx_service_checker/ckanext/hdx_service_checker/config/config.json'

ckanext-hdx_service_checker/ckanext/hdx_service_checker/config_declaration.yaml

@@ -0,0 +1,8 @@
+version: 1
+groups:
+  - annotation: Ignore Plugin
+    options:
+      - key: hdx.ignore.plugin.hdx_service_checker
+###############################################################################
+#  All options are in the config_declarations.yml of the hdx_theme extension  
+###############################################################################