HDX-10191 org stats download as xlsx

ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/organization_helper.py

@@ -1,20 +1,20 @@
-'''
+"""
 Created on Jan 14, 2015
 
 @author: alexandru-m-g
-'''
+"""
 
 import json
 import logging
 import os
 import six
-
+import openpyxl
 import ckanext.hdx_search.cli.click_feature_search_command as lunr
-import ckanext.hdx_theme.helpers.helpers as h
 import ckanext.hdx_users.helpers.mailer as hdx_mailer
 from sqlalchemy import func
 import ckanext.hdx_org_group.helpers.static_lists as static_lists
-
+from flask import make_response
+from tempfile import NamedTemporaryFile
 import ckan.lib.dictization as dictization
 import ckan.lib.dictization.model_dictize as model_dictize
 import ckan.lib.dictization.model_save as model_save
@@ -30,6 +30,8 @@
 from ckan.common import _, c, config
 import ckan.plugins.toolkit as toolkit
 import ckan.lib.base as base
+import ckanext.hdx_theme.util.jql as jql
+from openpyxl.styles import Alignment, Font
 
 BUCKET = str(uploader.get_storage_path()) + '/storage/uploads/group/'
 abort = base.abort
@@ -46,7 +48,7 @@
 
 
 def filter_and_sort_results_case_insensitive(results, sort_by, q=None, has_datasets=False):
-    '''
+    """
     :param results: list of organizations to filter/sort
     :type results: list[dict]
     :param sort_by:
@@ -57,7 +59,7 @@ def filter_and_sort_results_case_insensitive(results, sort_by, q=None, has_datas
     :type has_datasets: bool
     :return: sorted/filtered list
     :rtype: list[dict]
-    '''
+    """
 
     filtered_results = results
     if q:
@@ -348,9 +350,9 @@ def hdx_organization_delete(context, data_dict):
 
 
 def _run_core_group_org_action(context, data_dict, core_action):
-    '''
+    """
     Runs core ckan action with lunr update
-    '''
+    """
     test = True if config.get('ckan.site_id') == 'test.ckan.net' else False
     result = core_action(context, data_dict)
     if not test:
@@ -366,7 +368,7 @@ def hdx_group_or_org_update(context, data_dict, is_org=False):
     id = _get_or_bust(data_dict, 'id')
 
     group = model.Group.get(id)
-    context["group"] = group
+    context['group'] = group
     if group is None:
         raise NotFound('Group was not found.')
 
@@ -604,7 +606,7 @@ def hdx_group_or_org_create(context, data_dict, is_org=False):
         # to ensure they still work
         try:
             group_plugin.check_data_dict(data_dict, schema)
-        except TypeError as e:
+        except TypeError:
             group_plugin.check_data_dict(data_dict)
 
     data, errors = lib_plugins.plugin_validate(
@@ -666,8 +668,8 @@ def hdx_group_or_org_create(context, data_dict, is_org=False):
 
     if not context.get('defer_commit'):
         model.repo.commit()
-    context["group"] = group
-    context["id"] = group.id
+    context['group'] = group
+    context['id'] = group.id
 
     # creator of group/org becomes an admin
     # this needs to be after the repo.commit or else revisions break
@@ -718,9 +720,9 @@ def notify_admins(data_dict):
             # for admin in data_dict.get('admins'):
             hdx_mailer.mail_recipient(data_dict.get('admins'), data_dict.get('subject'), data_dict.get('message'))
     except Exception as e:
-        log.error("Email server error: can not send email to admin users" + e.message)
+        log.error('Email server error: can not send email to admin users' + e.message)
         return False
-    log.info("admin users where notified by email")
+    log.info('admin users where notified by email')
     return True
 
 
@@ -775,11 +777,11 @@ def _find_last_update_for_orgs(org_names):
             'model': model,
             'session': model.Session
         }
-        filter = 'organization:({}) +dataset_type:dataset'.format(' OR '.join(org_names))
+        fq_filter = 'organization:({}) +dataset_type:dataset'.format(' OR '.join(org_names))
 
         data_dict = {
             'q': '',
-            'fq': filter,
+            'fq': fq_filter,
             'fq_list': ['{!collapse field=organization nullPolicy=expand sort="metadata_modified desc"} '],
             'rows': len(org_names),
             'start': 0,
@@ -799,3 +801,63 @@ def org_add_last_updated_field(displayed_orgs):
 def hdx_organization_type_get_value(org_type_key):
     return next((org_type[0] for org_type in static_lists.ORGANIZATION_TYPE_LIST if org_type[1] == org_type_key),
                 org_type_key)
+
+def hdx_generate_organization_stats(org_dict):
+
+    # Define variable to load the dataframe
+    wb = openpyxl.Workbook()
+
+    # Bold font style
+    bold_font = Font(bold=True)
+
+    # Create SheetOne with Data
+    sheet_one = wb.active
+    sheet_one.title = 'Downloads and Page Views'
+
+    result = jql.pageviews_downloads_per_organization_last_5_years(org_dict.get('id'))
+    data = [('Date', 'Page View - Unique', 'Page Views - Total', 'Resource Download - Unique', 'Resource Download - Total')]
+    for key, value in result.items():
+        data.append((key, value.get('pageviews_unique'), value.get('pageviews_total'), value.get('downloads_unique'), value.get('downloads_total')))
+
+    for row_num, row_data in enumerate(data, start=1):
+        for col_num, cell_value in enumerate(row_data, start=1):
+            cell = sheet_one.cell(row=row_num, column=col_num, value=cell_value)
+            if row_num == 1:
+                cell.font = bold_font  # Apply bold to header row
+            cell.alignment = Alignment(horizontal='center', vertical='center')
+
+    # Set the width of the columns for the second sheet
+    for col_letter in ['A', 'B', 'C', 'D', 'E']:
+        sheet_one.column_dimensions[col_letter].width = 25
+
+    # Create SheetTwo with Data
+    sheet_two = wb.create_sheet(title='README')
+
+    data = [('Overview', 'This spreadsheet contains the number of downloads of files and page views of datasets of the organization, tracked monthly over the past 4 years.'),
+            ('Data Source', 'The data has been sourced from the analytics platform Mixpanel [https://mixpanel.com/].'),
+            ('Contents', 'The spreadsheet includes the following information: \n1. Page Views: Total page views by month. \n2. Downloads: Total number of downloads by month.'),
+            ('Caveats', 'To ensure accurate data representation, we have excluded as much bot traffic as possible.'),
+            ('Update Frequency', 'The spreadsheet is refreshed automatically on the first day of each month.'),
+            ('Contact', 'For additional inquiries, please contact us at [email protected]')]
+
+    # Add data to the worksheet
+    for row_num, (header, text) in enumerate(data, start=1):
+        sheet_two[f'A{row_num}'] = header
+        sheet_two[f'A{row_num}'].font = bold_font  # Apply bold to the first column
+        sheet_two[f'B{row_num}'] = text
+        sheet_two[f'A{row_num}'].alignment = Alignment(horizontal='left', vertical='top')
+        sheet_two[f'B{row_num}'].alignment = Alignment(horizontal='left', vertical='top', wrap_text=True)
+
+    # Set the width of the columns
+    sheet_two.column_dimensions['A'].width = 20
+    sheet_two.column_dimensions['B'].width = 100
+
+    # Iterate the loop to read the cell values
+    with NamedTemporaryFile() as tmp:
+        wb.save(tmp)
+        tmp.seek(0)
+        output = make_response(tmp.read())
+        output.headers['Content-Type'] = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+        output.headers['Content-Disposition'] = f'attachment; filename="{org_dict.get("name")}_stats.xlsx"'
+
+    return output

ckanext-hdx_org_group/ckanext/hdx_org_group/plugin.py

@@ -92,7 +92,7 @@ def group_types(self):
 
     # IGroupForm
     def setup_template_variables(self, context, data_dict):
-        org.new_org_template_variables(context, data_dict)
+        org.new_org_template_variables(data_dict)
 
     # IValidators
     def get_validators(self):

ckanext-hdx_org_group/ckanext/hdx_org_group/views/organization.py

@@ -2,19 +2,19 @@
 
 from flask import Blueprint
 from six.moves.urllib.parse import urlencode
-
+from ckan.types import Context
+import ckan.lib.plugins as lib_plugins
 import ckan.model as model
 import ckan.plugins.toolkit as tk
-import ckan.lib.plugins as lib_plugins
-
 import ckanext.hdx_org_group.helpers.org_meta_dao as org_meta_dao
 import ckanext.hdx_org_group.helpers.organization_helper as helper
 import ckanext.hdx_org_group.helpers.static_lists as static_lists
 import ckanext.hdx_theme.helpers.helpers as hdx_helpers
-
-from ckan.views.group import _get_group_template, CreateGroupView, EditGroupView
+from ckan.views.group import CreateGroupView, EditGroupView, _get_group_template
 from ckanext.hdx_org_group.controller_logic.organization_read_logic import OrgReadLogic
-from ckanext.hdx_org_group.controller_logic.organization_stats_logic import OrganizationStatsLogic
+from ckanext.hdx_org_group.controller_logic.organization_stats_logic import (
+    OrganizationStatsLogic,
+)
 from ckanext.hdx_org_group.views.light_organization import _index
 from ckanext.hdx_theme.util.light_redirect import check_redirect_needed
 from ckanext.hdx_theme.util.mail import NoRecipientException
@@ -46,7 +46,7 @@ def index():
 
 @check_redirect_needed
 def read(id):
-    context = {
+    context: Context = {
         'model': model,
         'session': model.Session,
         'for_view': True,
@@ -82,19 +82,19 @@ def read(id):
             }
             template_file = _get_group_template('read_template', 'organization')
             return render(template_file, template_data)
-    except NotFound as e:
+    except NotFound:
         abort(404, _('Page not found'))
-    except NotAuthorized as e:
+    except NotAuthorized:
         abort(403, _('Not authorized to see this page'))
 
 
 def _generate_template_data_for_custom_org(org_read_logic):
-    '''
+    """
     :param org_read_logic:
     :type org_read_logic: OrgReadLogic
     :returns: the template data dict
     :rtype: dict
-    '''
+    """
     org_meta = org_read_logic.org_meta
     org_dict = org_meta.org_dict
     org_id = org_dict['id']
@@ -147,7 +147,7 @@ def _generate_template_data_for_custom_org(org_read_logic):
 
             },
 
-            # This is hear for compatibility with the custom_org_header.html template, which is still
+            # This is here for compatibility with the custom_org_header.html template, which is still
             # used from pylon controllers
             'org_meta': {
                 'id': org_dict['name'],
@@ -177,7 +177,7 @@ def _generate_template_data_for_custom_org(org_read_logic):
 
 
 def request_new():
-    context = {'model': model, 'session': model.Session, 'user': g.user}
+    context: Context = {'model': model, 'session': model.Session, 'user': g.user}
     try:
         check_access('hdx_send_new_org_request', context)
     except NotAuthorized:
@@ -231,7 +231,7 @@ def _process_new_org_request():
 
     if hdx_org_type_code:
         hdx_org_type = next(
-            (type[0] for type in static_lists.ORGANIZATION_TYPE_LIST if type[1] == hdx_org_type_code), '-1')
+            (_type[0] for _type in static_lists.ORGANIZATION_TYPE_LIST if _type[1] == hdx_org_type_code), '-1')
 
     data = {
         'name': request.form.get('name', ''),
@@ -271,7 +271,7 @@ def _transform_dict_for_mailing(data_dict):
     return data_dict_for_mailing
 
 
-def new_org_template_variables(context, data_dict):
+def new_org_template_variables(data_dict):
     data_dict['hdx_org_type_list'] = [{'value': '-1', 'text': _('-- Please select --')}] + \
                               [{'value': t[1], 'text': _(t[0])} for t in static_lists.ORGANIZATION_TYPE_LIST]
 
@@ -296,15 +296,15 @@ def stats(id):
 
 
 def restore(id):
-    context = {
+    context: Context = {
         'model': model, 'session': model.Session,
         'user': g.user,
         'for_edit': True,
     }
 
     try:
         check_access('organization_patch', context, {'id': id})
-    except NotAuthorized as e:
+    except NotAuthorized:
         return abort(403, _('Unauthorized to restore this organization'))
 
     try:
@@ -329,7 +329,7 @@ def activity(id):
 
 
 def activity_offset(id, offset=0):
-    '''
+    """
      Modified core functionality to use the new OrgMetaDao class
     for fetching information needed on all org-related pages.
 
@@ -340,7 +340,7 @@ def activity_offset(id, offset=0):
     :param offset:
     :type offset: int
     :return:
-    '''
+    """
     org_meta = org_meta_dao.OrgMetaDao(id, g.user, g.userobj)
     org_meta.fetch_all()
     org_dict = org_meta.org_dict
@@ -350,26 +350,53 @@ def activity_offset(id, offset=0):
 
     # Add the group's activity stream (already rendered to HTML) to the
     # template context for the group/read.html template to retrieve later.
-    context = {'model': model, 'session': model.Session,
+    context: Context = {'model': model, 'session': model.Session,
                'user': g.user, 'for_view': True}
     group_activity_stream = get_action('organization_activity_list')(
         context, {'id': org_dict['id'], 'offset': offset})
 
-
-
     extra_vars = {
         'org_dict': org_dict,
         'org_meta': org_meta,
         'group_activity_stream': group_activity_stream,
 
     }
-    template = None
     if org_meta.is_custom:
         template = 'organization/custom_activity_stream.html'
     else:
         template = lib_plugins.lookup_group_plugin('organization').activity_template()
     return render(template, extra_vars)
 
+def organization_stats(id):
+    """
+        Handles downloading .xlsx organization stats
+
+        :returns: xlsx
+    """
+
+    context: Context = {
+        'model': model,
+        'session': model.Session,
+        'user': g.user or g.author,
+        'auth_user_obj': g.userobj
+    }
+
+    try:
+        check_access('organization_update', context, {'id': id})
+    except NotAuthorized:
+        return abort(403, _('Unauthorized to restore this organization'))
+
+    # check if organization exists
+    try:
+        org_dict = get_action('organization_show')(context, {'id': id})
+        output = helper.hdx_generate_organization_stats(org_dict)
+        return output
+
+    except NotFound:
+        return abort(404, _('Organization not found'))
+    except NotAuthorized:
+        return abort(404, _('Organization not found'))
+
 
 hdx_org.add_url_rule(u'/', view_func=index, strict_slashes=False)
 hdx_org.add_url_rule(
@@ -387,3 +414,4 @@ def activity_offset(id, offset=0):
 hdx_org.add_url_rule(u'/restore/<id>', view_func=restore, methods=[u'POST'])
 hdx_org.add_url_rule(u'/activity/<id>', view_func=activity)
 hdx_org.add_url_rule(u'/activity/<id>/<int:offset>', view_func=activity_offset, defaults={'offset': 0})
+hdx_org.add_url_rule(u'/<id>/download_stats', view_func=organization_stats)