From 6a54dc8b4b38f80339f13133c180e1c9be6b0b82 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Wed, 5 Jan 2022 17:12:25 -0300 Subject: [PATCH 01/11] =?UTF-8?q?adiciona=20cr=5Fj1=20em=20verifica=C3=A7?= =?UTF-8?q?=C3=A3o=20de=20report=5Fid?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/adapter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/adapter.py b/api/adapter.py index f505dea..4f4fc20 100644 --- a/api/adapter.py +++ b/api/adapter.py @@ -8,6 +8,8 @@ def wrapper_mount_json_for_report(report_id, result_query, attrs): return mount_json_for_reports_tr_j1(result_query, attrs) if report_id == 'tr_j4': return mount_json_for_reports_tr_j4(result_query, attrs) + if report_id == 'cr_j1': + return mount_json_for_reports_cr_j1(result_query, attrs) def mount_json_for_reports(result_query): From 234c349b544a8db982f641f8e841407d12b7457c Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Wed, 5 Jan 2022 17:12:39 -0300 Subject: [PATCH 02/11] adiciona adapter para gerar report cr_j1 --- api/adapter.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/api/adapter.py b/api/adapter.py index 4f4fc20..c94d0e1 100644 --- a/api/adapter.py +++ b/api/adapter.py @@ -301,3 +301,67 @@ def mount_json_for_reports_ir_a1(result_query_reports_ir_a1, attrs): json_results['Report_Items'] = [ri for ri in report_items.values() if ri['Item']] return json_results + + +def mount_json_for_reports_cr_j1(result_query_reports_cr_j1, attrs): + json_results = { + "Report_Header": { + "Created": datetime.now().isoformat(), + "Created_By": "Scientific Electronic Library Online SUSHI API", + "Customer_ID": attrs.get('customer', ''), + "Report_ID": attrs.get('report_data', {}).report_id, + "Release": attrs.get('report_data', {}).release, + "Report_Name": attrs.get('report_data', {}).name, + "Institution_Name": attrs.get('institution_name', ''), + "Institution_ID": [{ + "Type": "ISNI", + "Value": attrs.get('institution_id', '') + }], + }, + "Report_Filters": [{ + "Name": "Begin_Date", + "Value": attrs.get('begin_date', '') + }, { + "Name": "End_Date", + "Value": attrs.get('end_date', '') + }], + "Report_Attributes": [{ + "Name": "Attributes_To_Show", + "Value": "Data_Type|Access_Method" + }], + "Exceptions": [], + "Report_Items": [] + } + + report_items = {} + + for r in result_query_reports_cr_j1: + r_collection_acronym = attrs.get('collection') + if r_collection_acronym not in report_items: + report_items[r_collection_acronym] = { + 'Title': r_collection_acronym, + 'Item_ID': [], + 'Platform': attrs.get('platform', ''), + 'Data_Type': 'Collection', + 'Section_Type': 'Journal', + 'Access_Type': 'Open Access', + 'Access_Method': 'Regular', + 'Performance': []} + + for m in ['Total_Item_Requests', 'Unique_Item_Requests']: + metric_name = m[0].lower() + m[1:].replace('_', '') + + performance_m = { + 'Period': { + 'Begin_Date': str(r.beginDate), + 'End_Date': str(r.endDate) + }, + 'Instance': { + 'Metric_Type': m, + 'Count': str(getattr(r, metric_name)) + } + } + report_items[r_collection_acronym]['Performance'].append(performance_m) + + json_results['Report_Items'] = [ri for ri in report_items.values() if ri['Title']] + return json_results From 9dd6543cafa3741f473fabee29d3a72755e19a03 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Wed, 5 Jan 2022 17:13:01 -0300 Subject: [PATCH 03/11] adiciona chamadas de procedures de report cr_j1 --- api/db_calls.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/db_calls.py b/api/db_calls.py index 4beaae1..cfec412 100644 --- a/api/db_calls.py +++ b/api/db_calls.py @@ -17,6 +17,10 @@ DB_CALL_V2_TR_J1_JOURNAL_MONTHLY = 'CALL V2_TR_J1_JOURNAL_MONTHLY("%s", "%s", "%s", "%s")' DB_CALL_V2_TR_J1_TOTALS = 'CALL V2_TR_J1_TOTALS("%s", "%s", "%s")' DB_CALL_V2_TR_J1_MONTHLY = 'CALL V2_TR_J1_MONTHLY("%s", "%s", "%s")' +DB_CALL_V2_CR_J1_TOTALS = 'CALL V2_CR_J1_TOTALS("%s", "%s", "%s", "%s")' +DB_CALL_V2_CR_J1_MONTHLY = 'CALL V2_CR_J1_MONTHLY("%s", "%s", "%s", "%s")' + + REPORT_ID_STATUS_COLUMN_DICT = { 'ir_a1': 'status_sushi_article_metric', 'tr_j1': 'status_sushi_journal_metric', From 16657db0ba3f8f46343d2ba90db0c2070cbca1db Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Wed, 5 Jan 2022 17:13:18 -0300 Subject: [PATCH 04/11] =?UTF-8?q?adiciona=20report=20cr=5Fj1=20em=20tratam?= =?UTF-8?q?ento=20de=20par=C3=A2metros=20de=20url?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/db_calls.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/db_calls.py b/api/db_calls.py index cfec412..b52e7b9 100644 --- a/api/db_calls.py +++ b/api/db_calls.py @@ -71,6 +71,7 @@ }, 'global': { 'tr_j1': (DB_CALL_V2_TR_J1_TOTALS, ['begin_date', 'end_date', 'collection']), + 'cr_j1': (DB_CALL_V2_CR_J1_TOTALS, ['begin_date', 'end_date', 'collection', 'collection_extra']), } }, 'monthly': { @@ -79,6 +80,7 @@ }, 'global': { 'tr_j1': (DB_CALL_V2_TR_J1_MONTHLY, ['begin_date', 'end_date', 'collection']), + 'cr_j1': (DB_CALL_V2_CR_J1_MONTHLY, ['begin_date', 'end_date', 'collection', 'collection_extra']), } } } From 19aa550f6c8c0359602e8ccacb6891c651c4efa3 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Wed, 5 Jan 2022 17:13:46 -0300 Subject: [PATCH 05/11] =?UTF-8?q?adiciona=20descri=C3=A7=C3=A3o=20para=20r?= =?UTF-8?q?elat=C3=B3rio=20cr=5Fj1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/static/json/counter_report.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/api/static/json/counter_report.json b/api/static/json/counter_report.json index 0b7ae81..cba4d32 100644 --- a/api/static/json/counter_report.json +++ b/api/static/json/counter_report.json @@ -14,6 +14,13 @@ "Report_Description": "Platform-level usage summarized by Metric_Type.", "Path": "/reports/pr_p1" }, + "cr_j1": { + "Report_Name": "Collection Usage", + "Report_ID": "cr_j1", + "Release": "5", + "Report_Description": "Collection-level usage summarized by Metric_Type.", + "Path": "/reports/cr_j1" + }, "dr": { "Report_Name": "Database Master Report", "Report_ID": "dr", From d961b69e986d0b59f329c88cb915a22c350ee0fd Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Wed, 5 Jan 2022 17:14:03 -0300 Subject: [PATCH 06/11] =?UTF-8?q?adiciona=20procedure=20que=20cria=20relat?= =?UTF-8?q?=C3=B3rio=20cr=5Fj1=20total?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/static/sql/procedures.sql | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/api/static/sql/procedures.sql b/api/static/sql/procedures.sql index 75a9e2c..d4b77af 100644 --- a/api/static/sql/procedures.sql +++ b/api/static/sql/procedures.sql @@ -534,3 +534,20 @@ BEGIN WHERE cjc.collection = collection; END $$ DELIMITER ; + +DELIMITER $$ +CREATE PROCEDURE V2_CR_J1_TOTALS(IN beginDate date, IN endDate date, IN collection varchar(3), IN collection_extra varchar(3)) +BEGIN + SELECT + MIN(year_month_day) AS beginDate, + MAX(year_month_day) AS endDate, + SUM(total_item_requests) AS totalItemRequests, + SUM(unique_item_requests) AS uniqueItemRequests + FROM sushi_journal_metric sjm + JOIN counter_journal cj on sjm.idjournal_sjm = cj.id + WHERE + (sjm.collection in (collection , collection_extra)) AND + (year_month_day between beginDate AND endDate) AND + (cj.online_issn <> '' OR cj.print_issn <> ''); +END $$ +DELIMITER ; From 6b7a0cb526cb8d2f53115fe647022f152f0fc53d Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Wed, 5 Jan 2022 17:14:18 -0300 Subject: [PATCH 07/11] =?UTF-8?q?adiciona=20procedure=20que=20cria=20relat?= =?UTF-8?q?=C3=B3rio=20cr=5Fj1=20agregando=20resultados=20por=20m=C3=AAs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/static/sql/procedures.sql | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/api/static/sql/procedures.sql b/api/static/sql/procedures.sql index d4b77af..1544c4c 100644 --- a/api/static/sql/procedures.sql +++ b/api/static/sql/procedures.sql @@ -551,3 +551,23 @@ BEGIN (cj.online_issn <> '' OR cj.print_issn <> ''); END $$ DELIMITER ; + +DELIMITER $$ +CREATE PROCEDURE V2_CR_J1_MONTHLY(IN beginDate date, IN endDate date, IN collection varchar(3), IN collection_extra varchar(3)) +BEGIN + SELECT + SUBSTR(sjm.year_month_day, 1, 7) AS yearMonth, + MIN(year_month_day) AS beginDate, + MAX(year_month_day) AS endDate, + SUM(total_item_requests) AS totalItemRequests, + SUM(unique_item_requests) AS uniqueItemRequests + FROM sushi_journal_metric sjm + JOIN counter_journal cj on sjm.idjournal_sjm = cj.id + WHERE + (sjm.collection in (collection, collection_extra)) AND + (year_month_day between beginDate AND endDate) AND + (online_issn <> '' OR print_issn <> '') + GROUP BY + yearMonth; +END $$ +DELIMITER ; From 2f0f707da6e8a32ac60642a9bda148ae2559df30 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Wed, 5 Jan 2022 17:14:35 -0300 Subject: [PATCH 08/11] =?UTF-8?q?adiciona=20m=C3=A9todo=20auxiliar=20para?= =?UTF-8?q?=20limpeza=20de=20campos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/utils.py | 8 ++++++++ api/views.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/api/utils.py b/api/utils.py index 5dde1c2..22f7ddd 100644 --- a/api/utils.py +++ b/api/utils.py @@ -42,3 +42,11 @@ def is_valid_date_format(date): if re.match(pattern=REGEX_DATE_FORMAT, string=date): return True return False + + +def clean_field(field): + els = [] + for c in field: + if c not in ('\"', '\'', '\n', '\t', ' '): + els.append(c) + return ''.join(els) diff --git a/api/views.py b/api/views.py index 505dc2a..94872e7 100644 --- a/api/views.py +++ b/api/views.py @@ -15,7 +15,7 @@ from .lib.database import get_dates_not_ready from .models import DBSession from .sql_declarative import Status, Alert, Member, Report -from .utils import handle_str_date, is_valid_date_range, is_valid_issn, is_valid_date_format +from .utils import handle_str_date, is_valid_date_range, is_valid_issn, is_valid_date_format, clean_field from .values import collection_acronym_to_collection_name From c4ad2fe95a78433b7267e759d1f98dc7080fce32 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Wed, 5 Jan 2022 17:15:33 -0300 Subject: [PATCH 09/11] =?UTF-8?q?melhora=20m=C3=A9todo=20que=20verifica=20?= =?UTF-8?q?se=20relat=C3=B3rio=20n=C3=A3o=20contem=20dados=20de=20acesso?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/views.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/api/views.py b/api/views.py index 94872e7..469f503 100644 --- a/api/views.py +++ b/api/views.py @@ -123,7 +123,7 @@ def reports_report_id(self): json_metrics = _wrapper_call_report(report_id, attrs) # Caso não existam dados de acesso para o período selecionado - if len(json_metrics.get('Report_Items', [])) == 0: + if is_empty_report(json_metrics.get('Report_Items', [])): return error_no_usage_available() # Obtém exceções @@ -150,6 +150,23 @@ def check_exceptions(params, begin_date, end_date, report_id, collection): return exceptions +def is_empty_report(report_items): + if len(report_items) == 0: + return True + + for item in report_items: + for performance in item.get('Performance', []): + try: + count = int(performance.get('Instance', {}).get('Count', 0)) + except ValueError: + count = 0 + + if count > 0: + return False + + return True + + def _wrapper_call_report(report_id, attrs): granularity, mode = _get_granularity_and_mode(attrs) if attrs['api'] == 'v2': From 08f2685ae428669f6cb9b11a06561df18c052534 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Wed, 5 Jan 2022 17:16:02 -0300 Subject: [PATCH 10/11] =?UTF-8?q?adiciona=20tratamento=20para=20lidar=20co?= =?UTF-8?q?m=20m=C3=BAltiplos=20acr=C3=B4nimos=20para=20uma=20mesma=20cole?= =?UTF-8?q?=C3=A7=C3=A3o=20(nbr=20e=20scl)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/views.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/api/views.py b/api/views.py index 469f503..bf6a440 100644 --- a/api/views.py +++ b/api/views.py @@ -113,6 +113,8 @@ def reports_report_id(self): 'api': clean_field(api_version), } + _set_collection_extra(report_id, attrs) + try: report_data = DBSession.query(Report).filter_by(report_id=report_id).one() except NoResultFound or MultipleResultsFound: @@ -181,6 +183,8 @@ def _wrapper_call_report(report_id, attrs): p_value = attrs.get(p) if p_value: params.append(p_value) + if not p_value and p == 'collection_extra': + params.append('') result_query_metrics = DBSession.execute(procedure_name % tuple(params)) return wrapper_mount_json_for_report(report_id, result_query_metrics, attrs) @@ -201,6 +205,16 @@ def _get_granularity_and_mode(attrs): return granularity, mode +def _set_collection_extra(report_id, attrs): + if report_id == 'cr_j1': + if attrs['collection'] == 'scl': + attrs.update({'collection_extra': 'nbr'}) + if attrs['collection'] == 'nbr': + attrs.update({'collection_extra': 'scl'}) + else: + attrs.update({'collection_extra': ''}) + + def _check_filters(params): invalid_filters = [] From 872936b764af690d5977caa96e2f71dd9532f194 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Wed, 5 Jan 2022 17:16:24 -0300 Subject: [PATCH 11/11] corrige dados em initialize_db --- api/initialize_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/initialize_db.py b/api/initialize_db.py index 4398575..e2cf4b8 100644 --- a/api/initialize_db.py +++ b/api/initialize_db.py @@ -43,7 +43,7 @@ def main(argv=sys.argv): Base.metadata.create_all(engine) DBSession.configure(bind=engine) - file_counter_report = os.path.join(os.getcwd(), '/app/static/counter_report.json') + file_counter_report = os.path.join(os.getcwd(), 'app/static/counter_report.json') with transaction.manager: with open(file_counter_report) as f: application_url = settings.get('application.url')