HDX-9406 - upgrade python 3.10 and docker image

Dockerfile

@@ -1,4 +1,8 @@
-FROM ubuntu:focal
+#for local env
+#FROM ubuntu:jammy
+
+#for local MACOS env
+FROM --platform=linux/arm64/v8 ubuntu:jammy
 # FROM public.ecr.aws/unocha/debian-base-s6:11-slim
 
 ARG S6_VERSION=v2.2.0.3
@@ -52,35 +56,40 @@ RUN apt-get -qq -y update && \
     # make python3 and pip3 available as python and pip
     ln -sf /usr/bin/python3 /usr/bin/python && \
     ln -sf /usr/bin/pip3 /usr/bin/pip && \
-    # perpare nginx unit installation
+    # prepare nginx unit installation
     curl --output /usr/share/keyrings/nginx-keyring.gpg https://unit.nginx.org/keys/nginx-keyring.gpg && \
-    echo "deb [signed-by=/usr/share/keyrings/nginx-keyring.gpg] https://packages.nginx.org/unit/ubuntu/ focal unit" > /etc/apt/sources.list.d/unit.list && \
-    echo "deb-src [signed-by=/usr/share/keyrings/nginx-keyring.gpg] https://packages.nginx.org/unit/ubuntu/ focal unit" >> /etc/apt/sources.list.d/unit.list && \
+    echo "deb [signed-by=/usr/share/keyrings/nginx-keyring.gpg] https://packages.nginx.org/unit/ubuntu/ jammy unit" > /etc/apt/sources.list.d/unit.list && \
+    echo "deb-src [signed-by=/usr/share/keyrings/nginx-keyring.gpg] https://packages.nginx.org/unit/ubuntu/ jammy unit" >> /etc/apt/sources.list.d/unit.list && \
     # install nginx unit
     apt-get -qq -y update && \
     apt-get -qq -y install \
-      unit unit-python3.8 && \
+      unit unit-python3.10 && \
     ln -sf /usr/lib/unit/modules/python3.8.unit.so /usr/lib/unit/modules/python.unit.so && \
     # prepare files and folders
     mkdir -p /var/log/ckan /srv/filestore /srv/webassets /etc/services.d/unit /etc/ckan && \
     cd /srv/ckan && \
     python -m pip install --upgrade pip && \
     pip install -r requirement-setuptools.txt && \
-    pip install --upgrade -r requirements.txt && \
+    #pip install --upgrade -r requirements.txt && \
+    pip install pip-tools==7.3.0 && \
     pip install --upgrade -r requirements-hdxckantool.txt && \
+    pip-sync requirements.txt && \
     pip install \
       elastic-apm[flask] \
       newrelic && \
     chmod +x run_pytest_with_coverage.sh && \
     chmod +x setup_py_helper.sh && \
     ./setup_py_helper.sh && \
-    newrelic-admin generate-config LICENSE_KEY /srv/newrelic.ini && \
+#    #newrelic-admin generate-config LICENSE_KEY /srv/newrelic.ini && \
     chown -R www-data ckan/public/base/i18n && \
     cp -a docker/run_unit /etc/services.d/unit/run && \
     chown www-data:www-data -R /var/log/ckan /srv/filestore && \
     cp -a docker/hdxckantool-ng.py /srv/hdxckantool-ng.py && \
     chmod +x /srv/hdxckantool-ng.py && \
     ln -s /srv/hdxckantool-ng.py /usr/sbin/hdxckantool && \
+#    cp -a docker/ckancli /srv/ckancli && \
+#    chmod +x /srv/ckancli && \
+#    ln -s /srv/ckancli /usr/sbin/ckan && \
     echo "application/vnd.geo+json       geojson" >> /etc/mime.types && \
     apt-get -qq -y remove \
         build-essential \
@@ -91,8 +100,7 @@ RUN apt-get -qq -y update && \
         libxml2-dev \
         libxslt1-dev \
         libyaml-dev \
-        zlib1g-dev \
-        python-dev && \
+        zlib1g-dev && \
     apt-get -y autoremove && \
     apt-get clean && \
     rm -rf \
@@ -109,7 +117,8 @@ RUN apt-get -qq -y update && \
     curl -o /tmp/s6-overlay.tar.gz -jkSL https://github.com/just-containers/s6-overlay/releases/download/${S6_VERSION}/s6-overlay-${S6_ARCH}.tar.gz && \
     tar xzf /tmp/s6-overlay.tar.gz -C / && \
     rm -f /tmp/s6-overlay.tar.gz && \
-    ln -sf /usr/bin/sh /bin/sh
+    ln -sf /usr/bin/sh /bin/sh && \
+    ln -sf /usr/bin/bash /bin/bash
 
 
 VOLUME ["/srv/filestore", "/srv/backup", "/var/log/ckan"]

ckanext-hdx_package/ckanext/hdx_package/actions/create.py

@@ -7,6 +7,7 @@
 import logging
 
 import six
+
 from flask import request
 
 import ckan.lib.plugins as lib_plugins
@@ -18,6 +19,8 @@
 import ckanext.hdx_package.helpers.resource_triggers.common
 import ckanext.hdx_package.helpers.resource_triggers.geopreview as geopreview
 import ckanext.hdx_package.helpers.helpers as helpers
+from ckan.types import Context, DataDict, Schema
+from ckan.types.logic import ActionResult
 from ckanext.hdx_org_group.helpers.org_batch import get_batch_or_generate
 from ckanext.hdx_package.actions.update import process_batch_mode, flag_if_file_uploaded, run_action_without_geo_preview
 from ckanext.hdx_package.helpers.constants import BATCH_MODE, BATCH_MODE_DONT_GROUP
@@ -37,6 +40,7 @@
 @geopreview.geopreview_4_resources
 def resource_create(context, data_dict):
     '''
+
     This runs the 'resource_create' action from core ckan's create.py
     It allows us to do some minor changes and wrap it.
     '''
@@ -83,7 +87,8 @@ def resource_create(context, data_dict):
 @ckanext.hdx_package.helpers.resource_triggers.common.trigger_4_resource_changes(
     BEFORE_PACKAGE_UPDATE_LISTENERS, AFTER_PACKAGE_UPDATE_LISTENERS, VERSION_CHANGE_ACTIONS
 )
-def package_create(context, data_dict):
+def package_create(
+        context: Context, data_dict: DataDict) -> ActionResult.PackageCreate:
     '''Create a new dataset (package).
 
     You must be authorized to create new datasets. If you specify any groups
@@ -144,6 +149,19 @@ def package_create(context, data_dict):
         dictionary should have keys ``'key'`` (a string), ``'value'`` (a
         string)
     :type extras: list of dataset extra dictionaries
+    :param plugin_data: private package data belonging to plugins.
+        Only sysadmin users may set this value. It should be a dict that can
+        be dumped into JSON, and plugins should namespace their data with the
+        plugin name to avoid collisions with other plugins, eg::
+
+            {
+                "name": "test-dataset",
+                "plugin_data": {
+                    "plugin1": {"key1": "value1"},
+                    "plugin2": {"key2": "value2"}
+                }
+            }
+    :type plugin_data: dict
     :param relationships_as_object: see :py:func:`package_relationship_create`
         for the format of relationship dictionaries (optional)
     :type relationships_as_object: list of relationship dictionaries
@@ -190,10 +208,8 @@ def package_create(context, data_dict):
     else:
         package_plugin = lib_plugins.lookup_package_plugin(data_dict['type'])
 
-    if 'schema' in context:
-        schema = context['schema']
-    else:
-        schema = package_plugin.create_package_schema()
+    schema: Schema = context.get(
+        'schema') or package_plugin.create_package_schema()
 
     _check_access('package_create', context, data_dict)
 
@@ -233,16 +249,19 @@ def package_create(context, data_dict):
         model.Session.rollback()
         raise logic.ValidationError(errors)
 
+    plugin_data = data.get('plugin_data', False)
+    include_plugin_data = False
     if user:
 
         user_obj = model.User.by_name(six.ensure_text(user))
         if user_obj:
             data['creator_user_id'] = user_obj.id
+            include_plugin_data = user_obj.sysadmin and plugin_data
 
     # Replace model_save.package_dict_save() call with our wrapped version to be able to save groups
     # pkg = model_save.package_dict_save(data, context)
     from ckanext.hdx_package.actions.update import modified_save
-    pkg = modified_save(context, data)
+    pkg = modified_save(context, data,include_plugin_data)
 
     # pkg = model_save.package_dict_save(data, context)
 
@@ -263,7 +282,7 @@ def package_create(context, data_dict):
     for item in plugins.PluginImplementations(plugins.IPackageController):
         item.create(pkg)
 
-        item.after_create(context, data)
+        item.after_dataset_create(context, data)
 
     # Make sure that a user provided schema is not used in create_views
     # and on package_show
@@ -277,15 +296,15 @@ def package_create(context, data_dict):
             {'package': data})
 
     # Create activity
-    if pkg.type == 'dataset':
-        user_obj = model.User.by_name(user)
-        if user_obj:
-            user_id = user_obj.id
-        else:
-            user_id = 'not logged in'
-
-        activity = pkg.activity_stream_item('new', user_id)
-        session.add(activity)
+    # if pkg.type == 'dataset':
+    #     user_obj = model.User.by_name(user)
+    #     if user_obj:
+    #         user_id = user_obj.id
+    #     else:
+    #         user_id = 'not logged in'
+    #
+    #     activity = pkg.activity_stream_item('new', user_id)
+    #     session.add(activity)
 
     if not context.get('defer_commit'):
         model.repo.commit()
@@ -296,7 +315,8 @@ def package_create(context, data_dict):
         return pkg.id
 
     return _get_action('package_show')(
-        context.copy(), {'id': pkg.id}
+        context.copy(),
+        {'id': pkg.id, 'include_plugin_data': include_plugin_data}
     )
 
 

ckanext-hdx_package/ckanext/hdx_package/actions/update.py

@@ -11,6 +11,7 @@
 from six import text_type
 from flask import request
 from sqlalchemy import or_
+from typing import Any
 
 import ckan.lib.dictization.model_save as model_save
 import ckan.lib.munge as munge
@@ -19,9 +20,12 @@
 import ckan.logic.action.update as core_update
 import ckan.plugins as plugins
 import ckan.plugins.toolkit as tk
+import ckan.model as model
 import ckanext.hdx_package.helpers.resource_triggers.common
 import ckanext.hdx_package.helpers.resource_triggers.geopreview as geopreview
 import ckanext.hdx_package.helpers.helpers as helpers
+from ckan.types.logic import ActionResult
+from ckan.types import Context, DataDict
 from ckan.common import _
 from ckanext.hdx_org_group.helpers.org_batch import get_batch_or_generate
 from ckanext.hdx_package.helpers.analytics import QACompletedAnalyticsSender
@@ -176,12 +180,17 @@ def _fetch_prev_resources_info(model, resource_ids):
 @ckanext.hdx_package.helpers.resource_triggers.common.trigger_4_resource_changes(
     BEFORE_PACKAGE_UPDATE_LISTENERS, AFTER_PACKAGE_UPDATE_LISTENERS, VERSION_CHANGE_ACTIONS
 )
-def package_update(context, data_dict):
+def package_update(
+        context: Context, data_dict: DataDict) -> ActionResult.PackageUpdate:
     '''Update a dataset (package).
 
     You must be authorized to edit the dataset and the groups that it belongs
     to.
 
+    .. note:: Update methods may delete parameters not explicitly provided in the
+        data_dict. If you want to edit only a specific attribute use `package_patch`
+        instead.
+
     It is recommended to call
     :py:func:`ckan.logic.action.get.package_show`, make the desired changes to
     the result, and then call ``package_update()`` with it.
@@ -231,12 +240,9 @@ def package_update(context, data_dict):
 
     user = context['user']
     # get the schema
-    package_plugin = lib_plugins.lookup_package_plugin(pkg.type)
-    if 'schema' in context:
-        schema = context['schema']
-    else:
-        schema = package_plugin.update_package_schema()
 
+    package_plugin = lib_plugins.lookup_package_plugin(pkg.type)
+    schema = context.get('schema') or package_plugin.update_package_schema()
     if 'api_version' not in context:
         # check_data_dict() is deprecated. If the package_plugin has a
         # check_data_dict() we'll call it, if it doesn't have the method we'll
@@ -303,15 +309,21 @@ def package_update(context, data_dict):
         model.Session.rollback()
         raise ValidationError(errors)
 
-    # avoid revisioning by updating directly
+    #avoid revisioning by updating directly
     model.Session.query(model.Package).filter_by(id=pkg.id).update(
         {"metadata_modified": datetime.datetime.utcnow()})
     model.Session.refresh(pkg)
 
+    include_plugin_data = False
+    user_obj = context.get('auth_user_obj')
+    if user_obj:
+        plugin_data = data.get('plugin_data', False)
+        include_plugin_data = user_obj.sysadmin and plugin_data
+
     if 'tags' in data:
         data['tags'] = helpers.get_tag_vocabulary(data['tags'])
 
-    pkg = modified_save(context, data)
+    pkg = modified_save(context, data, include_plugin_data)
     # pkg = model_save.package_dict_save(data, context)
 
     context_org_update = context.copy()
@@ -324,7 +336,7 @@ def package_update(context, data_dict):
     # Needed to let extensions know the new resources ids
     model.Session.flush()
     for index, (resource, upload) in enumerate(
-        zip(data.get('resources', []), resource_uploads)):
+            zip(data.get('resources', []), resource_uploads)):
         resource['id'] = pkg.resources[index].id
 
         if upload:
@@ -335,18 +347,18 @@ def package_update(context, data_dict):
     for item in plugins.PluginImplementations(plugins.IPackageController):
         item.edit(pkg)
 
-        item.after_update(context, data)
+        item.after_dataset_update(context, data)
 
     # Create activity
-    if pkg.type == 'dataset':
-        user_obj = model.User.by_name(user)
-        if user_obj:
-            user_id = user_obj.id
-        else:
-            user_id = 'not logged in'
-
-        activity = pkg.activity_stream_item('changed', user_id)
-        session.add(activity)
+    # if pkg.type == 'dataset':
+    #     user_obj = model.User.by_name(user)
+    #     if user_obj:
+    #         user_id = user_obj.id
+    #     else:
+    #         user_id = 'not logged in'
+    #
+    #     activity = pkg.activity_stream_item('changed', user_id)
+    #     session.add(activity)
 
     if not context.get('defer_commit'):
         model.repo.commit()
@@ -360,7 +372,7 @@ def package_update(context, data_dict):
 
     # we could update the dataset so we should still be able to read it.
     context['ignore_auth'] = True
-    new_data_dict = _get_action('package_show')(context, {'id': data_dict['id']})
+    new_data_dict = _get_action('package_show')(context, {'id': data_dict['id'], "include_plugin_data": include_plugin_data})
 
     # HDX - delete previous files if needed
     for resource_dict in new_data_dict.get('resources'):
@@ -410,18 +422,20 @@ def process_skip_validation(context, data_dict):
         del data_dict[SKIP_VALIDATION]
 
 
-def modified_save(context, data):
+def modified_save(
+        context: Context, data: dict[str, Any],
+        include_plugin_data: bool = False) -> 'model.Package':
     """
     Wrapper around lib.dictization.model_save.package_dict_save
     """
     groups_key = 'groups'
     if groups_key in data:
         temp_groups = data[groups_key]
         data[groups_key] = None
-        pkg = model_save.package_dict_save(data, context)
+        pkg = model_save.package_dict_save(data, context, include_plugin_data)
         data[groups_key] = temp_groups
     else:
-        pkg = model_save.package_dict_save(data, context)
+        pkg = model_save.package_dict_save(data, context, include_plugin_data)
     package_membership_list_save(data.get("groups"), pkg, context)
     return pkg
 

ckanext-hdx_theme/ckanext/hdx_theme/helpers/helpers.py

@@ -687,7 +687,7 @@ def hdx_popular(type_, number, min=1, title=None):
 
 
 def hdx_license_list():
-    license_touple_list = base.model.Package.get_license_options()
+    license_touple_list = model.Package.get_license_options()
     license_dict_list = [{'value': _id, 'text': _title} for _title, _id in license_touple_list]
     return license_dict_list
 

ckanext-hdx_theme/ckanext/hdx_theme/templates/user/api_tokens.html

@@ -8,6 +8,7 @@
       <div class="col-8">
         {% if h.check_access('api_token_create', {'user': user['name']}) %}
           <form method="POST" class="form">
+            {{ h.csrf_input() }}
             <legend>{{ _('API Tokens') }}</legend>
             {% block errors %}{{ form.errors(error_summary) }}{% endblock %}
 

ckanext-hdx_theme/docs/dependencies/index.rst

@@ -24,9 +24,11 @@ Compilation steps
 
     *  FOR PYTHON 3::
 
-        pip install pip-tools==6.6.0
         pip install pip-tools==6.10.0
 
+        #for python 3.10
+        pip install pip-tools==7.3.0
+
 #. Compile the main dependencies:
 
    *  FOR PYTHON 2::

common-config-ini.txt

@@ -101,6 +101,8 @@ ckan.site_id = default
 
 ckan.plugins = hdx_dataviz dcat dcat_json_interface structured_data expire_api_token hdx_hxl_preview ytp_request hdx_pages hdx_choropleth_map_view hdx_geopreview_view hdx_chart_views hdx_service_checker hdx_analytics hdx_search sitemap hdx_org_group hdx_group hdx_package hdx_user_extra hdx_mail_validate hdx_users hdx_theme security requestdata s3filestore showcase stats resource_proxy text_view recline_view datastore activity
 
+
+
 ckan.views.default_views = image_view recline_view text_view
 
 ## Front-End Settings