From d9ae060bc19097d3ed69d916fdf951010542dc87 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Fri, 1 Apr 2022 11:09:01 +0200 Subject: [PATCH 01/63] Reuse markdownlint validator for restructuredtext-lint --- tooling.json | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tooling.json b/tooling.json index e538a24..2d25e44 100644 --- a/tooling.json +++ b/tooling.json @@ -269,7 +269,9 @@ ], "reporting": { "validator": "markdownlint", - "requirement_level": "RECOMMENDED" + "requirement_level": "RECOMMENDED", + "doc_file_type": "Markdown", + "doc_file_standard": "markdownlint" } } }, @@ -293,9 +295,10 @@ } ], "reporting": { - "validator": "json_not_empty", + "validator": "markdownlint", "requirement_level": "RECOMMENDED", - "subcriterion": "QC.Doc02.X" + "doc_file_type": "reStructuredText", + "doc_file_standard": "restructuredtext-lint" } } }, From a6f2ba09b7356f4de699f9155b7442510a6309c8 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Wed, 23 Mar 2022 17:05:57 +0100 Subject: [PATCH 02/63] Add kubectl for SvcQC.Dep --- tooling.json | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tooling.json b/tooling.json index 2d25e44..44400ca 100644 --- a/tooling.json +++ b/tooling.json @@ -327,6 +327,20 @@ } } }, + "Kubernetes": { + "kubectl": { + "docs": "https://kubernetes.io/docs/reference/kubectl/", + "docker": { + "dockerfile": "bitnami/kubectl:1.23", + "reviewed": "2022-03-23" + }, + "args": [ + { + "type": "positional", "description": "YAML deployment files", "value": "", "format": "string", "selectable": true, "repeatable": true + } + ] + } + }, "default": { "commands": { "docs": "https://indigo-dc.github.io/jenkins-pipeline-library/2.0.0/user/config_file.html#commands", @@ -439,6 +453,16 @@ "tools": { "Python": ["tox"] } + }, + "SvcQC.Dep": { + "description": { + "msg": "Perform the service deployment in an automated fashion following Infrastructure as Code (IaC) templates", + "improves": "immutable infrastructure deployment and maintenance", + "docs": "https://eosc-synergy.github.io/service-qa-baseline/#dep" + }, + "tools": { + "Kubernetes": ["kubectl"] + } } } } From 6cd29a7c923ecd122f7fdf356d81fb31d8fa2207 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Wed, 23 Mar 2022 17:23:27 +0100 Subject: [PATCH 03/63] Add 'native_template' property for kubectl --- tooling.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tooling.json b/tooling.json index 44400ca..305d7db 100644 --- a/tooling.json +++ b/tooling.json @@ -334,6 +334,7 @@ "dockerfile": "bitnami/kubectl:1.23", "reviewed": "2022-03-23" }, + "native_template": "kubectl", "args": [ { "type": "positional", "description": "YAML deployment files", "value": "", "format": "string", "selectable": true, "repeatable": true From 80461a32b20e9e0a95321b089a53ecc80eac1c41 Mon Sep 17 00:00:00 2001 From: ferag Date: Tue, 5 Apr 2022 15:12:03 +0200 Subject: [PATCH 04/63] FAIR EVA without curl --- QC.FAIR/Dockerfile | 4 +++ QC.FAIR/fair-eva.py | 75 +++++++++++++++++++++++++++++++++++++++++++++ tooling.json | 18 ++++++----- 3 files changed, 90 insertions(+), 7 deletions(-) create mode 100644 QC.FAIR/Dockerfile create mode 100755 QC.FAIR/fair-eva.py diff --git a/QC.FAIR/Dockerfile b/QC.FAIR/Dockerfile new file mode 100644 index 0000000..9c631a1 --- /dev/null +++ b/QC.FAIR/Dockerfile @@ -0,0 +1,4 @@ +FROM ferag/fair_eva:latest +LABEL maintainer="Fernando Aguilar " +COPY fair-eva.py /usr/bin +RUN chmod +x /usr/bin/fair-eva.py diff --git a/QC.FAIR/fair-eva.py b/QC.FAIR/fair-eva.py new file mode 100755 index 0000000..3dce1c8 --- /dev/null +++ b/QC.FAIR/fair-eva.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python + +import argparse +import json +import requests +from pathlib import Path + + +# HTTP methods +HTTP_METHODS = [ + 'POST', + 'PUT', + 'GET' +] +# Follows GitHub supported markup languages +# -- https://github.com/github/markup/blob/master/README.md#markups +AVAILABLE_HEADERS = [ + 'application/json' +] +# Ordered list of possible file locations +LOCATIONS = [ + '.', + 'docs', + '.github' +] + + +def get_input_args(): + parser = argparse.ArgumentParser(description=( + 'Prepare requests for FAIR assessment tool' + )) + parser.add_argument( + '-H', + metavar='HEADER', + type=str, + choices=AVAILABLE_HEADERS, + help='Header for the HTTP request' + ) + parser.add_argument( + '-X', + metavar='METHOD', + choices=HTTP_METHODS, + type=str, + help='HTTP method' + ) + parser.add_argument( + '-d', + metavar='DATA', + type=json.loads, + help='Data to submit in HTTP request (dict: {id: item_id, repo: oai-pmh, oai_base: oai_endpoint, lang: ES})' + ) + parser.add_argument( + '--tool_endpoint', + metavar='ENDPOINT', + type=str, + help='Enpoint to perform HTTP request. Example: http://localhost:9090/v1.0/rda/rda_all' + ) + + return parser.parse_args() + + +def main(): + args = get_input_args() + url = args.tool_endpoint + headers = {'Content-type': args.H} + data = args.d + if args.X == 'POST': + r = requests.post(url,data=json.dumps(data), headers=headers) + elif args.X == 'PUT': + r = requests.put(url,data=json.dumps(data), headers=headers) + else: + r = requests.get(url,data=json.dumps(data), headers=headers) + return json.dumps(r.text) + +print(main()) diff --git a/tooling.json b/tooling.json index 305d7db..2129615 100644 --- a/tooling.json +++ b/tooling.json @@ -138,19 +138,23 @@ } }, "fair": { - "fair-evaluator": { - "docs": "https://github.com/EOSC-synergy/FAIR_eva", + "fair-eva.py": { + "docs": "QC.FAIR/fair-eva.py", "docker": { - "image": "ferag/fair_eva:fair", + "image": "QC.FAIR/Dockerfile", "oneshot": false, - "reviewed": "2021-07-06" + "reviewed": "2022-04-05" }, - "executable": "/bin/curl", "args": [ - {"type": "optional", "format": "string", "description": "", "option": "-H", "value": "'Content-Type: application/json'", "selectable": true}, + {"type": "optional", "format": "string", "description": "", "option": "-H", "value": "application/json", "selectable": true}, {"type": "optional", "format": "string", "description": "", "option": "-X", "value": "POST", "selectable": true}, {"type": "optional", "format": "string", "description": "", "option": "-d", "value": "'{\"id\": \"10261/157765\", \"repo\": \"oai-pmh\", \"oai_base\": \"http://digital.csic.es/dspace-oai/request\"}'", "selectable": true}, - {"type": "positional", "format": "string", "description": "Endpoint", "value": "http://localhost:9090/v1.0/rda/rda_all", "selectable": true}] + {"type": "optional", "format": "string", "description": "Endpoint", "option": "--tool_endpoint", "value": "http://localhost:9090/v1.0/rda/rda_all", "selectable": true}], + "reporting": { + "validator": "fair-eva", + "threshold": 0, + "requirement_level": "REQUIRED" + } }, "F-UJI": { "docs": "https://github.com/pangaea-data-publisher/fuji/blob/master/README.md", From 6a107b6147ea52f415526dfc9238ba778310061c Mon Sep 17 00:00:00 2001 From: Fernando Aguilar Date: Tue, 12 Apr 2022 14:09:00 +0200 Subject: [PATCH 05/63] Update tooling.json PR fixes --- tooling.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tooling.json b/tooling.json index 2129615..9d1ec5f 100644 --- a/tooling.json +++ b/tooling.json @@ -138,10 +138,11 @@ } }, "fair": { - "fair-eva.py": { + "fair-evaluator": { + "executable" : "fair-eva.py", "docs": "QC.FAIR/fair-eva.py", "docker": { - "image": "QC.FAIR/Dockerfile", + "dockerfile": "QC.FAIR/Dockerfile", "oneshot": false, "reviewed": "2022-04-05" }, From 5d38a42129b4ce4163cfc3c2ac555b1f0d6b5e88 Mon Sep 17 00:00:00 2001 From: Fernando Aguilar Date: Wed, 13 Apr 2022 10:31:03 +0200 Subject: [PATCH 06/63] Update fair-eva.py --- QC.FAIR/fair-eva.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/QC.FAIR/fair-eva.py b/QC.FAIR/fair-eva.py index 3dce1c8..f46d4b3 100755 --- a/QC.FAIR/fair-eva.py +++ b/QC.FAIR/fair-eva.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 import argparse import json From c826f34886939981b1fb8b05810d2f9700cba005 Mon Sep 17 00:00:00 2001 From: Fernando Aguilar Date: Wed, 13 Apr 2022 13:41:54 +0200 Subject: [PATCH 07/63] Update Dockerfile --- QC.FAIR/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/QC.FAIR/Dockerfile b/QC.FAIR/Dockerfile index 9c631a1..0e59413 100644 --- a/QC.FAIR/Dockerfile +++ b/QC.FAIR/Dockerfile @@ -2,3 +2,4 @@ FROM ferag/fair_eva:latest LABEL maintainer="Fernando Aguilar " COPY fair-eva.py /usr/bin RUN chmod +x /usr/bin/fair-eva.py +CMD /FAIR_eva/start.sh From ada7700a0e973f1ab886ef6d02f09e961b1ef308 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Wed, 19 Jan 2022 10:05:35 +0100 Subject: [PATCH 08/63] Set choices for restructuredtext-lint (test dropdown list rendering) --- tooling.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tooling.json b/tooling.json index 9d1ec5f..6dfc781 100644 --- a/tooling.json +++ b/tooling.json @@ -290,10 +290,14 @@ "executable": "rst-lint", "args": [ { - "type": "optional", "description": "Minimum error level to report (default: 'warning')", "option": "--level", "value": "warning", "selectable": false + "type": "optional", "description": "Minimum error level to report (default: 'warning')", "option": "--level", "value": [ + "warning", "error", "severe", "info", "debug" + ], "selectable": true }, { - "type": "optional", "description": "Format of the output (default: 'text')", "option": "--format", "value": "json", "selectable": false + "type": "optional", "description": "Format of the output (default: 'text')", "option": "--format", "value": [ + "json", "text" + ], "selectable": true }, { "type": "positional", "description": "File/folder to lint", "value": ".", "selectable": true, "format": "string" From edca8d685106124abb6d11c6f988485d016667f2 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Wed, 19 Jan 2022 10:09:19 +0100 Subject: [PATCH 09/63] Add format --- tooling.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tooling.json b/tooling.json index 6dfc781..b608016 100644 --- a/tooling.json +++ b/tooling.json @@ -292,12 +292,12 @@ { "type": "optional", "description": "Minimum error level to report (default: 'warning')", "option": "--level", "value": [ "warning", "error", "severe", "info", "debug" - ], "selectable": true + ], "selectable": true, "format": "array" }, { "type": "optional", "description": "Format of the output (default: 'text')", "option": "--format", "value": [ "json", "text" - ], "selectable": true + ], "selectable": true, "format": "array" }, { "type": "positional", "description": "File/folder to lint", "value": ".", "selectable": true, "format": "string" From 76ed7adbd3d36d34075f7385f71f6918a1f772fc Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Wed, 26 Jan 2022 13:10:06 +0100 Subject: [PATCH 10/63] Fix: selectable after format --- tooling.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tooling.json b/tooling.json index b608016..cfe3254 100644 --- a/tooling.json +++ b/tooling.json @@ -292,12 +292,12 @@ { "type": "optional", "description": "Minimum error level to report (default: 'warning')", "option": "--level", "value": [ "warning", "error", "severe", "info", "debug" - ], "selectable": true, "format": "array" + ], "format": "array", "selectable": true }, { "type": "optional", "description": "Format of the output (default: 'text')", "option": "--format", "value": [ "json", "text" - ], "selectable": true, "format": "array" + ], "format": "array", "selectable": true }, { "type": "positional", "description": "File/folder to lint", "value": ".", "selectable": true, "format": "string" From 1a50807ac00c299aa51e8c32acace5e7fd8b50c3 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Wed, 26 Jan 2022 13:10:51 +0100 Subject: [PATCH 11/63] find_doc_files' file_type is an array --- tooling.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tooling.json b/tooling.json index cfe3254..ef8406a 100644 --- a/tooling.json +++ b/tooling.json @@ -320,7 +320,9 @@ }, "args": [ { - "type": "optional", "description": "Type of file to look for in the code repository", "option": "--file-type", "value": "", "format": "string", "selectable": true, "repeatable": false + "type": "optional", "description": "Type of file to look for in the code repository", "option": "--file-type", "value": [ + "README", "CODE_OF_CONDUCT", "CONTRIBUTING" + ], "format": "array", "selectable": true, "repeatable": false }, { "type": "optional", "description": "Path to look for in the code repository", "option": "--path", "value": "", "format": "string", "selectable": true, "repeatable": false From a3f295867288ead6b7eac214032ea9385ef16ed7 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Wed, 13 Apr 2022 15:15:34 +0200 Subject: [PATCH 12/63] Convert bandit's confidence/severity-level args to array --- tooling.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tooling.json b/tooling.json index ef8406a..0f45612 100644 --- a/tooling.json +++ b/tooling.json @@ -77,10 +77,10 @@ "type": "optional", "description": "only show output in the case of an error", "option": "--quiet", "value": "", "format": "string", "selectable": false, "repeatable": false }, { - "type": "optional", "description": "report only issues of a given confidence level or higher", "option": "--confidence-level", "value": "high", "format": "string", "selectable": false, "repeatable": false + "type": "optional", "description": "report only issues of a given confidence level or higher", "option": "--confidence-level", "value": ["all", "low", "medium", "high"], "format": "array", "selectable": true, "repeatable": false }, { - "type": "optional", "description": "report only issues of a given severity level or higher", "option": "--severity-level", "value": "high", "format": "string", "selectable": false, "repeatable": false + "type": "optional", "description": "report only issues of a given severity level or higher", "option": "--severity-level", "value": ["all", "low", "medium", "high"], "format": "array", "selectable": true, "repeatable": false }, { "type": "optional", "description": "find and process files in subdirectories", "option": "--recursive", "format": "string", "selectable": false, "repeatable": false From c4957d27c77296c8d50b243186f130c664efa762 Mon Sep 17 00:00:00 2001 From: Fernando Aguilar Date: Mon, 25 Apr 2022 12:19:51 +0200 Subject: [PATCH 13/63] Update Dockerfile --- QC.FAIR/Dockerfile | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/QC.FAIR/Dockerfile b/QC.FAIR/Dockerfile index 0e59413..5f620c8 100644 --- a/QC.FAIR/Dockerfile +++ b/QC.FAIR/Dockerfile @@ -1,5 +1,23 @@ -FROM ferag/fair_eva:latest -LABEL maintainer="Fernando Aguilar " +FROM ubuntu:20.04 + +MAINTAINER Fernando Aguilar "aguilarf@ifca.unican.es" + +RUN apt-get update -y && \ + apt-get install -y curl python3-pip python3-dev git vim lsof + +RUN ls +RUN git clone https://github.com/EOSC-synergy/FAIR_eva.git + +WORKDIR /FAIR_eva + +RUN pip3 install -r requirements.txt + +EXPOSE 5000 9090 +RUN ls +COPY config.ini.template /FAIR_eva/config.ini +RUN cd /FAIR_eva +RUN chmod 777 start.sh +RUN cat start.sh COPY fair-eva.py /usr/bin RUN chmod +x /usr/bin/fair-eva.py CMD /FAIR_eva/start.sh From 8b7d2c9fc025cde762536a132d11221818792c48 Mon Sep 17 00:00:00 2001 From: Fernando Aguilar Date: Mon, 25 Apr 2022 12:39:57 +0200 Subject: [PATCH 14/63] Update Dockerfile --- QC.FAIR/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/QC.FAIR/Dockerfile b/QC.FAIR/Dockerfile index 5f620c8..6ec748e 100644 --- a/QC.FAIR/Dockerfile +++ b/QC.FAIR/Dockerfile @@ -14,7 +14,7 @@ RUN pip3 install -r requirements.txt EXPOSE 5000 9090 RUN ls -COPY config.ini.template /FAIR_eva/config.ini +RUN cp config.ini.template config.ini RUN cd /FAIR_eva RUN chmod 777 start.sh RUN cat start.sh From bb49e8a529f22a19115e586bed8b42289d7a373f Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Tue, 12 Apr 2022 16:57:33 +0200 Subject: [PATCH 15/63] Two different kubectl entries for kustomize and config files --- tooling.json | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/tooling.json b/tooling.json index 0f45612..fc940ed 100644 --- a/tooling.json +++ b/tooling.json @@ -339,16 +339,29 @@ } }, "Kubernetes": { - "kubectl": { - "docs": "https://kubernetes.io/docs/reference/kubectl/", + "kubectl (via kustomize)": { + "docs": "https://kubernetes.io/docs/tasks/manage-kubernetes-objects/kustomization/", "docker": { "dockerfile": "bitnami/kubectl:1.23", "reviewed": "2022-03-23" }, - "native_template": "kubectl", + "native_template": "kubectl_kustomize", "args": [ { - "type": "positional", "description": "YAML deployment files", "value": "", "format": "string", "selectable": true, "repeatable": true + "type": "positional", "description": "Kustomization directory", "value": "", "format": "string", "selectable": true, "repeatable": false + } + ] + }, + "kubectl (via configuration files)": { + "docs": "https://kubernetes.io/docs/tasks/manage-kubernetes-objects/declarative-config/", + "docker": { + "dockerfile": "bitnami/kubectl:1.23", + "reviewed": "2022-03-23" + }, + "native_template": "kubectl_config_files", + "args": [ + { + "type": "positional", "description": "File, directory or URL of the configuration file in YAML format", "value": "", "format": "string", "selectable": true, "repeatable": true } ] } @@ -473,7 +486,10 @@ "docs": "https://eosc-synergy.github.io/service-qa-baseline/#dep" }, "tools": { - "Kubernetes": ["kubectl"] + "Kubernetes": [ + "kubectl (via kustomize)", + "kubectl (via configuration files)" + ] } } } From f3a97c02cedc2275a1c28b525085acc6f37d48db Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Tue, 12 Apr 2022 17:07:31 +0200 Subject: [PATCH 16/63] native_template -> template --- tooling.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tooling.json b/tooling.json index fc940ed..efe8503 100644 --- a/tooling.json +++ b/tooling.json @@ -345,7 +345,7 @@ "dockerfile": "bitnami/kubectl:1.23", "reviewed": "2022-03-23" }, - "native_template": "kubectl_kustomize", + "template": "kubectl_kustomize", "args": [ { "type": "positional", "description": "Kustomization directory", "value": "", "format": "string", "selectable": true, "repeatable": false @@ -358,7 +358,7 @@ "dockerfile": "bitnami/kubectl:1.23", "reviewed": "2022-03-23" }, - "native_template": "kubectl_config_files", + "template": "kubectl_config_files", "args": [ { "type": "positional", "description": "File, directory or URL of the configuration file in YAML format", "value": "", "format": "string", "selectable": true, "repeatable": true From a47a6b8bf74cadebd5a51ee4fcf5877eedc28872 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Wed, 13 Apr 2022 08:37:07 +0200 Subject: [PATCH 17/63] Add timeout option from kubectl-rollout-status --- tooling.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tooling.json b/tooling.json index efe8503..4b8a8b7 100644 --- a/tooling.json +++ b/tooling.json @@ -348,7 +348,8 @@ "template": "kubectl_kustomize", "args": [ { - "type": "positional", "description": "Kustomization directory", "value": "", "format": "string", "selectable": true, "repeatable": false + "type": "positional", "description": "Kustomization directory", "value": "", "format": "string", "selectable": true, "repeatable": false, + "type": "optional", "description": "[from kubectl-rollout-status] The length of time to wait before ending watch, zero means never. Any other values should contain a corresponding time unit (e.g. 1s, 2m, 3h)", "option": "--timeout", "value": "0s", "format": "string", "selectable": true, "repeatable": false } ] }, @@ -361,7 +362,8 @@ "template": "kubectl_config_files", "args": [ { - "type": "positional", "description": "File, directory or URL of the configuration file in YAML format", "value": "", "format": "string", "selectable": true, "repeatable": true + "type": "positional", "description": "File, directory or URL of the configuration file in YAML format", "value": "", "format": "string", "selectable": true, "repeatable": true, + "type": "optional", "description": "[from kubectl-rollout-status] The length of time to wait before ending watch, zero means never. Any other values should contain a corresponding time unit (e.g. 1s, 2m, 3h)", "option": "--timeout", "value": "0s", "format": "string", "selectable": true, "repeatable": false } ] } From 7f0bfc6afbe2770eb1f82cb77ede9a1a8e0a9af1 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Wed, 13 Apr 2022 13:09:18 +0200 Subject: [PATCH 18/63] Set ID for kubectl tools --- tooling.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tooling.json b/tooling.json index 4b8a8b7..b0285b8 100644 --- a/tooling.json +++ b/tooling.json @@ -348,8 +348,8 @@ "template": "kubectl_kustomize", "args": [ { - "type": "positional", "description": "Kustomization directory", "value": "", "format": "string", "selectable": true, "repeatable": false, - "type": "optional", "description": "[from kubectl-rollout-status] The length of time to wait before ending watch, zero means never. Any other values should contain a corresponding time unit (e.g. 1s, 2m, 3h)", "option": "--timeout", "value": "0s", "format": "string", "selectable": true, "repeatable": false + "type": "positional", "description": "Kustomization directory", "id": "k8s_config_files", "value": "", "format": "string", "selectable": true, "repeatable": false, + "type": "optional", "description": "[from kubectl-rollout-status] The length of time to wait before ending watch, zero means never. Any other values should contain a corresponding time unit (e.g. 1s, 2m, 3h)", "id": "k8s_rollout_status_timeout", "option": "--timeout", "value": "0s", "format": "string", "selectable": true, "repeatable": false } ] }, @@ -362,8 +362,7 @@ "template": "kubectl_config_files", "args": [ { - "type": "positional", "description": "File, directory or URL of the configuration file in YAML format", "value": "", "format": "string", "selectable": true, "repeatable": true, - "type": "optional", "description": "[from kubectl-rollout-status] The length of time to wait before ending watch, zero means never. Any other values should contain a corresponding time unit (e.g. 1s, 2m, 3h)", "option": "--timeout", "value": "0s", "format": "string", "selectable": true, "repeatable": false + "type": "positional", "description": "File, directory or URL of the configuration file in YAML format", "id": "k8s_config_files", "value": "", "format": "string", "selectable": true, "repeatable": true } ] } From fcb32b1cd36ba893fd59673e60d801c5c16c891a Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Wed, 13 Apr 2022 13:09:37 +0200 Subject: [PATCH 19/63] Document and --- docs/tooling.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/tooling.md b/docs/tooling.md index 86e75d8..9eba054 100644 --- a/docs/tooling.md +++ b/docs/tooling.md @@ -28,6 +28,7 @@ The following table summarizes the properties that ought to be set in the tool d | `docs` | string (url) | URL to the tool's official documentation | :heavy_check_mark: | | `docker` | object | See [Docker](#docker-docker-property) section | :heavy_check_mark: | | `executable` | string | Name of the executable. The tool's name is used by default.
*This is only required when the executable to be used is different from the tool's name* | | +| `template` | string | Name of the template to use to compose the final command (not a concatenation of `args`)) | | | `args` | object | See [Arguments](#arguments-args-property) section | | | `reporting` | object | See [Reporting](#reporting-reporting-property) section | | @@ -51,6 +52,7 @@ The `args` property enables the definition of the arguments involved in the tool | -------- | ---- | ----------- | -------- | | `type` | string (enum) | Type of the argument. Choose between [`subcommand`, `positional`, `optional`] | :heavy_check_mark: | | `description` | string | Short description of what the tool does | :heavy_check_mark: | +| `id` | string | Identifier of the argument (used only when `template` property is defined) | :white_check_mark: | | `value` | *any type* | The value of the argument | | | `option` | string | The option name (*only applicable for optional arguments*) | :white_check_mark: (for optional arguments) | | `format` | string (enum) | (for API clients) the value's data type. Useful for graphical interfaces, provides the means to render the form elements (inputs, text areas, dropdowns, ..). Choose between [`string`, `array`] | :white_check_mark: (for API clients) | From 07970c6537c19aea78c73cdc4cedb903ae640218 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Mon, 25 Apr 2022 22:25:07 +0200 Subject: [PATCH 20/63] Fix: dockerfile -> image --- tooling.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tooling.json b/tooling.json index b0285b8..8ae692c 100644 --- a/tooling.json +++ b/tooling.json @@ -342,7 +342,7 @@ "kubectl (via kustomize)": { "docs": "https://kubernetes.io/docs/tasks/manage-kubernetes-objects/kustomization/", "docker": { - "dockerfile": "bitnami/kubectl:1.23", + "image": "bitnami/kubectl:1.23", "reviewed": "2022-03-23" }, "template": "kubectl_kustomize", @@ -356,7 +356,7 @@ "kubectl (via configuration files)": { "docs": "https://kubernetes.io/docs/tasks/manage-kubernetes-objects/declarative-config/", "docker": { - "dockerfile": "bitnami/kubectl:1.23", + "image": "bitnami/kubectl:1.23", "reviewed": "2022-03-23" }, "template": "kubectl_config_files", From 6382493aa3b8e3396e177b98d0688456521fe729 Mon Sep 17 00:00:00 2001 From: ferag Date: Wed, 27 Apr 2022 10:48:26 +0200 Subject: [PATCH 21/63] Improve on request FAIR_EVA args --- QC.FAIR/fair-eva.py | 55 ++++++++++++++------------------------------- tooling.json | 10 ++++----- 2 files changed, 22 insertions(+), 43 deletions(-) diff --git a/QC.FAIR/fair-eva.py b/QC.FAIR/fair-eva.py index f46d4b3..3a9b673 100755 --- a/QC.FAIR/fair-eva.py +++ b/QC.FAIR/fair-eva.py @@ -5,49 +5,28 @@ import requests from pathlib import Path - -# HTTP methods -HTTP_METHODS = [ - 'POST', - 'PUT', - 'GET' -] -# Follows GitHub supported markup languages -# -- https://github.com/github/markup/blob/master/README.md#markups -AVAILABLE_HEADERS = [ - 'application/json' -] -# Ordered list of possible file locations -LOCATIONS = [ - '.', - 'docs', - '.github' -] - - def get_input_args(): parser = argparse.ArgumentParser(description=( 'Prepare requests for FAIR assessment tool' )) parser.add_argument( - '-H', - metavar='HEADER', + '-ID', + metavar='ID', type=str, choices=AVAILABLE_HEADERS, - help='Header for the HTTP request' + help='Persistent Identifier' ) parser.add_argument( - '-X', - metavar='METHOD', - choices=HTTP_METHODS, + '-R', + metavar='PLUGIN', type=str, help='HTTP method' ) parser.add_argument( - '-d', - metavar='DATA', - type=json.loads, - help='Data to submit in HTTP request (dict: {id: item_id, repo: oai-pmh, oai_base: oai_endpoint, lang: ES})' + '-B', + metavar='OAI-PMH', + type=str, + help='OAI-PMH_ENDPOINT' ) parser.add_argument( '--tool_endpoint', @@ -62,14 +41,14 @@ def get_input_args(): def main(): args = get_input_args() url = args.tool_endpoint - headers = {'Content-type': args.H} - data = args.d - if args.X == 'POST': - r = requests.post(url,data=json.dumps(data), headers=headers) - elif args.X == 'PUT': - r = requests.put(url,data=json.dumps(data), headers=headers) - else: - r = requests.get(url,data=json.dumps(data), headers=headers) + headers = {'Content-Type':'application/json'} + data = { + "id": args.ID, + "repo": args.R, + "oai_base": args.B, + "lang": "ES" + } + r = requests.post(url,data=json.dumps(data), headers=headers) return json.dumps(r.text) print(main()) diff --git a/tooling.json b/tooling.json index 8ae692c..c30da68 100644 --- a/tooling.json +++ b/tooling.json @@ -138,7 +138,7 @@ } }, "fair": { - "fair-evaluator": { + "fair-eva": { "executable" : "fair-eva.py", "docs": "QC.FAIR/fair-eva.py", "docker": { @@ -147,10 +147,10 @@ "reviewed": "2022-04-05" }, "args": [ - {"type": "optional", "format": "string", "description": "", "option": "-H", "value": "application/json", "selectable": true}, - {"type": "optional", "format": "string", "description": "", "option": "-X", "value": "POST", "selectable": true}, - {"type": "optional", "format": "string", "description": "", "option": "-d", "value": "'{\"id\": \"10261/157765\", \"repo\": \"oai-pmh\", \"oai_base\": \"http://digital.csic.es/dspace-oai/request\"}'", "selectable": true}, - {"type": "optional", "format": "string", "description": "Endpoint", "option": "--tool_endpoint", "value": "http://localhost:9090/v1.0/rda/rda_all", "selectable": true}], + {"type": "optional", "format": "string", "description": "Persistent Identifier", "option": "-ID", "value": "10261/157765", "selectable": true}, + {"type": "optional", "format": "string", "description": "FAIR EVA plugin", "option": "-R", "value": "oai-pmh", "selectable": true}, + {"type": "optional", "format": "string", "description": "OAI-PMH Endpoint", "option": "-B", "value": "http://digital.csic.es/dspace-oai/request", "selectable": true}, + {"type": "optional", "format": "string", "description": "FAIR EVA endpoint", "option": "--tool_endpoint", "value": "http://localhost:9090/v1.0/rda/rda_all", "selectable": true}], "reporting": { "validator": "fair-eva", "threshold": 0, From 3585e3ecb10b122416be2740baadb419be6c6ff9 Mon Sep 17 00:00:00 2001 From: ferag Date: Wed, 27 Apr 2022 12:05:45 +0200 Subject: [PATCH 22/63] Change FAIR-evaluator to FAIR-eva --- tooling.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tooling.json b/tooling.json index c30da68..13d61f2 100644 --- a/tooling.json +++ b/tooling.json @@ -414,7 +414,7 @@ "docs": "https://indigo-dc.github.io/sqa-baseline/" }, "tools": { - "fair": ["fair-evaluator", "F-UJI"] + "fair": ["fair-eva", "F-UJI"] } }, "QC.Fun": { From 0cc0634654f5aed8e2a3deabe2530b682c167b72 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Wed, 27 Apr 2022 17:26:47 +0200 Subject: [PATCH 23/63] Fix CMD so FAIR-EVA starts gracefully --- QC.FAIR/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/QC.FAIR/Dockerfile b/QC.FAIR/Dockerfile index 6ec748e..bc5c867 100644 --- a/QC.FAIR/Dockerfile +++ b/QC.FAIR/Dockerfile @@ -20,4 +20,4 @@ RUN chmod 777 start.sh RUN cat start.sh COPY fair-eva.py /usr/bin RUN chmod +x /usr/bin/fair-eva.py -CMD /FAIR_eva/start.sh +CMD ["/FAIR_eva/start.sh", "/FAIR_eva/config.ini"] From fd8460436477e5f9994a69ca84f85ce3f9596bb1 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Wed, 27 Apr 2022 17:57:14 +0200 Subject: [PATCH 24/63] Fix: remove choices from -ID arg --- QC.FAIR/fair-eva.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/QC.FAIR/fair-eva.py b/QC.FAIR/fair-eva.py index 3a9b673..d42473f 100755 --- a/QC.FAIR/fair-eva.py +++ b/QC.FAIR/fair-eva.py @@ -5,6 +5,7 @@ import requests from pathlib import Path + def get_input_args(): parser = argparse.ArgumentParser(description=( 'Prepare requests for FAIR assessment tool' @@ -13,7 +14,6 @@ def get_input_args(): '-ID', metavar='ID', type=str, - choices=AVAILABLE_HEADERS, help='Persistent Identifier' ) parser.add_argument( @@ -34,7 +34,7 @@ def get_input_args(): type=str, help='Enpoint to perform HTTP request. Example: http://localhost:9090/v1.0/rda/rda_all' ) - + return parser.parse_args() From c284a52905252871222cdc783c5c82685c902879 Mon Sep 17 00:00:00 2001 From: Fernando Aguilar Date: Thu, 28 Apr 2022 11:01:25 +0200 Subject: [PATCH 25/63] Update Dockerfile --- QC.FAIR/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/QC.FAIR/Dockerfile b/QC.FAIR/Dockerfile index bc5c867..4919a69 100644 --- a/QC.FAIR/Dockerfile +++ b/QC.FAIR/Dockerfile @@ -20,4 +20,5 @@ RUN chmod 777 start.sh RUN cat start.sh COPY fair-eva.py /usr/bin RUN chmod +x /usr/bin/fair-eva.py +ENV CONFIG_FILE=/FAIR_eva/config.ini CMD ["/FAIR_eva/start.sh", "/FAIR_eva/config.ini"] From 75f25d49bac0f13df51ff9ab026fd2cd2d64ccf3 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Tue, 12 Apr 2022 15:13:14 +0200 Subject: [PATCH 26/63] Use digest for non-tagged images --- tooling.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tooling.json b/tooling.json index 13d61f2..a7bb8d9 100644 --- a/tooling.json +++ b/tooling.json @@ -41,7 +41,7 @@ "checkstyle": { "docs": "https://github.com/checkstyle/checkstyle", "docker": { - "image": "davrodgon/checkstyle", + "image": "davrodgon/checkstyle@sha256:2ed1d4efd817afde40b219057935139818ee6b88105e75010bde53cd65786bf2", "reviewed": "1970-01-01" }, "args": [{ @@ -124,7 +124,7 @@ "jepl_support": true, "docs": "https://tox.readthedocs.io/", "docker": { - "image": "painless/tox", + "image": "painless/tox@sha256:4f31b329507f9ab658e3c36c5252ad063cabe3d978d6836e0ba322ccb79af060", "reviewed": "1970-01-01" }, "args": [{ @@ -160,7 +160,7 @@ "F-UJI": { "docs": "https://github.com/pangaea-data-publisher/fuji/blob/master/README.md", "docker": { - "image": "steinsteiny/fuji:fair", + "image": "steinsteiny/fuji:v1.3.8.full", "oneshot": false, "reviewed": "2021-07-20" }, From 4b6ecace54842e8021668475ac9d007ec3575a14 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Tue, 26 Apr 2022 09:14:53 +0200 Subject: [PATCH 27/63] Jenkins credential ID for KUBECONFIG envvar as optional arg --- tooling.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tooling.json b/tooling.json index a7bb8d9..013cfad 100644 --- a/tooling.json +++ b/tooling.json @@ -349,7 +349,8 @@ "args": [ { "type": "positional", "description": "Kustomization directory", "id": "k8s_config_files", "value": "", "format": "string", "selectable": true, "repeatable": false, - "type": "optional", "description": "[from kubectl-rollout-status] The length of time to wait before ending watch, zero means never. Any other values should contain a corresponding time unit (e.g. 1s, 2m, 3h)", "id": "k8s_rollout_status_timeout", "option": "--timeout", "value": "0s", "format": "string", "selectable": true, "repeatable": false + "type": "optional", "description": "[from kubectl-rollout-status] The length of time to wait before ending watch, zero means never. Any other values should contain a corresponding time unit (e.g. 1s, 2m, 3h)", "id": "k8s_rollout_status_timeout", "option": "--timeout", "value": "0s", "format": "string", "selectable": true, "repeatable": false, + "type": "optional", "description": "Jenkins credential ID with the value of KUBECONFIG environment variable", "option": "--kubeconfig-jenkins-credential-id", "value": "kubernetes-cluster", "format": "string", "selectable": true, "repeatable": false } ] }, From cc3ed5de15b26d25fff9c2a68fdb5856326b0a7b Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Tue, 26 Apr 2022 11:13:46 +0200 Subject: [PATCH 28/63] Special kubectl args for setting Jenkins credentials --- docs/tooling.md | 7 ++++++- tooling.json | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/tooling.md b/docs/tooling.md index 9eba054..1b643d6 100644 --- a/docs/tooling.md +++ b/docs/tooling.md @@ -54,12 +54,17 @@ The `args` property enables the definition of the arguments involved in the tool | `description` | string | Short description of what the tool does | :heavy_check_mark: | | `id` | string | Identifier of the argument (used only when `template` property is defined) | :white_check_mark: | | `value` | *any type* | The value of the argument | | -| `option` | string | The option name (*only applicable for optional arguments*) | :white_check_mark: (for optional arguments) | +| `option` | string | The option name (*only applicable for optional arguments*). See [Special option values](#special-option-values) section for additional customization | :white_check_mark: (for optional arguments) | | `format` | string (enum) | (for API clients) the value's data type. Useful for graphical interfaces, provides the means to render the form elements (inputs, text areas, dropdowns, ..). Choose between [`string`, `array`] | :white_check_mark: (for API clients) | | `selectable` | boolean | (for API clients) Whether the argument's value shall be customized by the user, or otherwise it is a fixed (non-modifiable) value | :white_check_mark: (for API clients) | | `repeatable` | boolean | (for API clients) Whether the same argument can be used several times | :white_check_mark: (for API clients) | | `args` | object | Suitable when defining commands with more than one type of argument, it allows to define nested `args` properties | | +#### Special `option` values +The name of the `option` property might tell the SQAaaS API to perform additional work. So far it is only supported to set credentials when the `option` value contains the following substrings: +- `jenkins-credential-id`, such as in `--kubeconfig-jenkins-credential-id`. +- `jenkins-credential-variable`, such as in `--kubeconfig-jenkins-credential-variable`. + ### Reporting (`reporting` property) The `reporting` property provides the pointers to the suitable output validator that is capable of parsing the data produced by the given tool. The list of available validator plugins officially supported in the SQAaaS platform can be found in the [sqaaas-reporting-plugins](https://github.com/eosc-synergy/sqaaas-reporting-plugins) repository. diff --git a/tooling.json b/tooling.json index 013cfad..6fb6495 100644 --- a/tooling.json +++ b/tooling.json @@ -350,7 +350,8 @@ { "type": "positional", "description": "Kustomization directory", "id": "k8s_config_files", "value": "", "format": "string", "selectable": true, "repeatable": false, "type": "optional", "description": "[from kubectl-rollout-status] The length of time to wait before ending watch, zero means never. Any other values should contain a corresponding time unit (e.g. 1s, 2m, 3h)", "id": "k8s_rollout_status_timeout", "option": "--timeout", "value": "0s", "format": "string", "selectable": true, "repeatable": false, - "type": "optional", "description": "Jenkins credential ID with the value of KUBECONFIG environment variable", "option": "--kubeconfig-jenkins-credential-id", "value": "kubernetes-cluster", "format": "string", "selectable": true, "repeatable": false + "type": "optional", "description": "Jenkins credential ID with the value of KUBECONFIG environment variable", "option": "--kubeconfig-jenkins-credential-id", "value": "kubernetes-cluster", "format": "string", "selectable": true, "repeatable": false, + "type": "optional", "description": "KUBECONFIG environment variable name", "option": "--kubeconfig-jenkins-credential-variable", "value": "KUBECONFIG", "format": "string", "selectable": false, "repeatable": false } ] }, From fe602549a1cfb01327f254ac48cc89e3e999ae4c Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Tue, 26 Apr 2022 11:15:52 +0200 Subject: [PATCH 29/63] Add missing args for kubectl-with-yaml-config-files --- tooling.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tooling.json b/tooling.json index 6fb6495..5c7c6b2 100644 --- a/tooling.json +++ b/tooling.json @@ -364,7 +364,10 @@ "template": "kubectl_config_files", "args": [ { - "type": "positional", "description": "File, directory or URL of the configuration file in YAML format", "id": "k8s_config_files", "value": "", "format": "string", "selectable": true, "repeatable": true + "type": "positional", "description": "File, directory or URL of the configuration file in YAML format", "id": "k8s_config_files", "value": "", "format": "string", "selectable": true, "repeatable": true, + "type": "optional", "description": "[from kubectl-rollout-status] The length of time to wait before ending watch, zero means never. Any other values should contain a corresponding time unit (e.g. 1s, 2m, 3h)", "id": "k8s_rollout_status_timeout", "option": "--timeout", "value": "0s", "format": "string", "selectable": true, "repeatable": false, + "type": "optional", "description": "Jenkins credential ID with the value of KUBECONFIG environment variable", "option": "--kubeconfig-jenkins-credential-id", "value": "kubernetes-cluster", "format": "string", "selectable": true, "repeatable": false, + "type": "optional", "description": "KUBECONFIG environment variable name", "option": "--kubeconfig-jenkins-credential-variable", "value": "KUBECONFIG", "format": "string", "selectable": false, "repeatable": false } ] } From 01960898e84b28de32f791816f8916472aa80157 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Wed, 4 May 2022 12:55:15 +0200 Subject: [PATCH 30/63] Fix: each arg in its own dict entry for kubectl commands --- tooling.json | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tooling.json b/tooling.json index 5c7c6b2..3f19469 100644 --- a/tooling.json +++ b/tooling.json @@ -348,9 +348,15 @@ "template": "kubectl_kustomize", "args": [ { - "type": "positional", "description": "Kustomization directory", "id": "k8s_config_files", "value": "", "format": "string", "selectable": true, "repeatable": false, - "type": "optional", "description": "[from kubectl-rollout-status] The length of time to wait before ending watch, zero means never. Any other values should contain a corresponding time unit (e.g. 1s, 2m, 3h)", "id": "k8s_rollout_status_timeout", "option": "--timeout", "value": "0s", "format": "string", "selectable": true, "repeatable": false, - "type": "optional", "description": "Jenkins credential ID with the value of KUBECONFIG environment variable", "option": "--kubeconfig-jenkins-credential-id", "value": "kubernetes-cluster", "format": "string", "selectable": true, "repeatable": false, + "type": "positional", "description": "Kustomization directory", "id": "k8s_config_files", "value": "", "format": "string", "selectable": true, "repeatable": false + }, + { + "type": "optional", "description": "[from kubectl-rollout-status] The length of time to wait before ending watch, zero means never. Any other values should contain a corresponding time unit (e.g. 1s, 2m, 3h)", "id": "k8s_rollout_status_timeout", "option": "--timeout", "value": "0s", "format": "string", "selectable": true, "repeatable": false + }, + { + "type": "optional", "description": "Jenkins credential ID with the value of KUBECONFIG environment variable", "option": "--kubeconfig-jenkins-credential-id", "value": "kubernetes-cluster", "format": "string", "selectable": true, "repeatable": false + }, + { "type": "optional", "description": "KUBECONFIG environment variable name", "option": "--kubeconfig-jenkins-credential-variable", "value": "KUBECONFIG", "format": "string", "selectable": false, "repeatable": false } ] From b6bcd3c7e5c75f12310fbb0958b0abc15703c136 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Mon, 2 May 2022 13:54:27 +0200 Subject: [PATCH 31/63] Add type of criteria --- tooling.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tooling.json b/tooling.json index 3f19469..998062a 100644 --- a/tooling.json +++ b/tooling.json @@ -396,6 +396,7 @@ }, "criteria": { "QC.Acc": { + "type": "software", "description": { "msg": "Code publicly available and managed by a version control system", "improves": "collaboration, reproducibility", @@ -406,6 +407,7 @@ } }, "QC.Doc": { + "type": "software", "description": { "msg": "Treat documentation as code by using markup languages to automatically build and place it in online repositories", "improves": "outreach capacity, documentation maintenance", @@ -419,6 +421,7 @@ } }, "QC.FAIR": { + "type": "fair", "description": { "msg": "FAIRness level of Digital Objects.", "improves": "findability, accessibility, interoperability, reusability", @@ -429,6 +432,7 @@ } }, "QC.Fun": { + "type": "software", "description": { "msg": "Ensure compliance with the functional requirements to meet your users’ expectations", "improves": "end-user satisfaction, usability", @@ -439,6 +443,7 @@ } }, "QC.Lic": { + "type": "software", "description": { "msg": "Usage of an open license to distribute your code", "improves": "external contributions, reusability", @@ -449,6 +454,7 @@ } }, "QC.Met": { + "type": "software", "description": { "msg": "Make source code citable", "improves": "findability", @@ -460,6 +466,7 @@ } }, "QC.Sec": { + "type": "software", "description": { "msg": "Secure your software by finding (statically) common issues associated to the programming language in use and look for disclosed security vulnerabilities", "improves": "security issues detection", @@ -470,6 +477,7 @@ } }, "QC.Sty": { + "type": "software", "description": { "msg": "Use code style standards to guide your code writing so you let others understand it", "improves": "readability, reusability", @@ -482,6 +490,7 @@ } }, "QC.Uni": { + "type": "software", "description": { "msg": "Test the behaviour of segments or units within your code (e.g. conditionals, loops, functions)", "improves": "code design, early bug detection", @@ -492,6 +501,7 @@ } }, "SvcQC.Dep": { + "type": "service", "description": { "msg": "Perform the service deployment in an automated fashion following Infrastructure as Code (IaC) templates", "improves": "immutable infrastructure deployment and maintenance", From a5d0f1bac2b8673fa16f2db3e082e19ce2ff90f4 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Mon, 2 May 2022 13:56:55 +0200 Subject: [PATCH 32/63] QC.Fun -> SvcQC.Fun (fixes #61) --- tooling.json | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tooling.json b/tooling.json index 998062a..67d6ada 100644 --- a/tooling.json +++ b/tooling.json @@ -431,17 +431,6 @@ "fair": ["fair-eva", "F-UJI"] } }, - "QC.Fun": { - "type": "software", - "description": { - "msg": "Ensure compliance with the functional requirements to meet your users’ expectations", - "improves": "end-user satisfaction, usability", - "docs": "https://indigo-dc.github.io/sqa-baseline/#functional-testing-qc.fun" - }, - "tools": { - "Python": ["tox"] - } - }, "QC.Lic": { "type": "software", "description": { @@ -513,6 +502,17 @@ "kubectl (via configuration files)" ] } + }, + "SvcQC.Fun": { + "type": "service", + "description": { + "msg": "Ensure compliance with the functional requirements to meet your users’ expectations", + "improves": "end-user satisfaction, usability", + "docs": "https://indigo-dc.github.io/sqa-baseline/#functional-testing-qc.fun" + }, + "tools": { + "Python": ["tox"] + } } } } From 583923d26128c06c4f5632401a260b09a015f40f Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Mon, 2 May 2022 14:00:27 +0200 Subject: [PATCH 33/63] Fix: correct URL --- tooling.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tooling.json b/tooling.json index 67d6ada..05f8710 100644 --- a/tooling.json +++ b/tooling.json @@ -508,7 +508,7 @@ "description": { "msg": "Ensure compliance with the functional requirements to meet your users’ expectations", "improves": "end-user satisfaction, usability", - "docs": "https://indigo-dc.github.io/sqa-baseline/#functional-testing-qc.fun" + "docs": "https://eosc-synergy.github.io/service-qa-baseline/#fun" }, "tools": { "Python": ["tox"] From 64c74dd5b662ff5636a957aaf66cd1d61bd45bbb Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Mon, 9 May 2022 14:12:43 +0200 Subject: [PATCH 34/63] Script for getting git tags for the current commit. --- QC.Ver/Dockerfile | 6 ++++++ QC.Ver/get_git_tags.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 QC.Ver/Dockerfile create mode 100644 QC.Ver/get_git_tags.py diff --git a/QC.Ver/Dockerfile b/QC.Ver/Dockerfile new file mode 100644 index 0000000..325ac4c --- /dev/null +++ b/QC.Ver/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.9-alpine +LABEL maintainer="Pablo Orviz " +RUN apk --no-cache add build-base +RUN python3 -m pip install GitPython==3.1.27 +COPY get_git_tags.py /usr/bin +RUN chmod +x /usr/bin/get_git_tags.py diff --git a/QC.Ver/get_git_tags.py b/QC.Ver/get_git_tags.py new file mode 100644 index 0000000..bf192d8 --- /dev/null +++ b/QC.Ver/get_git_tags.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python + +import argparse +import git + + +def get_input_args(): + parser = argparse.ArgumentParser(description=( + 'Get git release tags associated with the current commit ID.' + )) + parser.add_argument( + '--repo-path', + type=str, + default='.', + help='Path to the git repository' + ) + + return parser.parse_args() + + +def main(): + args = get_input_args() + + repo = git.Repo(args.repo_path) + tags = repo.git.tag('-l', '--contains', 'HEAD') + tag_list = tags.split('\n') + + +print(main()) From fe8bc48a714cfa61f263196b8078a07c2e9e7cb4 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Tue, 10 May 2022 14:37:15 +0200 Subject: [PATCH 35/63] Add is_semver definition to QC.Ver --- tooling.json | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tooling.json b/tooling.json index 05f8710..bc1bf28 100644 --- a/tooling.json +++ b/tooling.json @@ -338,6 +338,19 @@ } } }, + "Semantic Versioning": { + "is_semver.py": { + "docs": "QC.Ver/is_semver.py", + "docker": { + "dockerfile": "QC.Ver/Dockerfile", + "reviewed": "2022-05-10" + }, + "reporting": { + "validator": "is_semver", + "requirement_level": "REQUIRED" + } + } + }, "Kubernetes": { "kubectl (via kustomize)": { "docs": "https://kubernetes.io/docs/tasks/manage-kubernetes-objects/kustomization/", @@ -489,6 +502,17 @@ "Python": ["tox"] } }, + "QC.Ver": { + "type": "software", + "description": { + "msg": "Software releases follow Semantic Versioning specification", + "improves": "dependency management", + "docs": "https://indigo-dc.github.io/sqa-baseline/#semantic-versioning-qc.ver" + }, + "tools": { + "SemVer": ["is_semver.py"] + } + }, "SvcQC.Dep": { "type": "service", "description": { From 9da72cb901f06e3c969143b5a0b5a3ab0793afc2 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Wed, 11 May 2022 13:47:23 +0200 Subject: [PATCH 36/63] Set FAIR-EVA as optional tool (to exclude from QAA) --- tooling.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tooling.json b/tooling.json index bc1bf28..7ffc287 100644 --- a/tooling.json +++ b/tooling.json @@ -154,7 +154,7 @@ "reporting": { "validator": "fair-eva", "threshold": 0, - "requirement_level": "REQUIRED" + "requirement_level": "OPTIONAL" } }, "F-UJI": { From 4f3628d3b3ced66bf7b4345bc501a0cc7de5fd2d Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Wed, 11 May 2022 17:00:21 +0200 Subject: [PATCH 37/63] Fix semver lang --- tooling.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tooling.json b/tooling.json index 7ffc287..9ae2c92 100644 --- a/tooling.json +++ b/tooling.json @@ -510,7 +510,7 @@ "docs": "https://indigo-dc.github.io/sqa-baseline/#semantic-versioning-qc.ver" }, "tools": { - "SemVer": ["is_semver.py"] + "Semantic Versioning": ["is_semver.py"] } }, "SvcQC.Dep": { From 6670bc3e5fa72dea6f3833609e0606fc17680a55 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Wed, 11 May 2022 18:47:45 +0200 Subject: [PATCH 38/63] Set default 'high' values for bandit's confidence and severity --- tooling.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tooling.json b/tooling.json index 9ae2c92..5edd279 100644 --- a/tooling.json +++ b/tooling.json @@ -77,10 +77,10 @@ "type": "optional", "description": "only show output in the case of an error", "option": "--quiet", "value": "", "format": "string", "selectable": false, "repeatable": false }, { - "type": "optional", "description": "report only issues of a given confidence level or higher", "option": "--confidence-level", "value": ["all", "low", "medium", "high"], "format": "array", "selectable": true, "repeatable": false + "type": "optional", "description": "report only issues of a given confidence level or higher", "option": "--confidence-level", "value": ["high", "all", "low", "medium"], "format": "array", "selectable": true, "repeatable": false }, { - "type": "optional", "description": "report only issues of a given severity level or higher", "option": "--severity-level", "value": ["all", "low", "medium", "high"], "format": "array", "selectable": true, "repeatable": false + "type": "optional", "description": "report only issues of a given severity level or higher", "option": "--severity-level", "value": ["high", "all", "low", "medium"], "format": "array", "selectable": true, "repeatable": false }, { "type": "optional", "description": "find and process files in subdirectories", "option": "--recursive", "format": "string", "selectable": false, "repeatable": false From fe9f2cfd1091ce9a60df911f0933feae2c957f78 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Thu, 12 May 2022 08:10:28 +0200 Subject: [PATCH 39/63] 'all' value will check all file types --- QC.Doc/find_doc_files.py | 5 ++++- tooling.json | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/QC.Doc/find_doc_files.py b/QC.Doc/find_doc_files.py index a426ad1..aee5aad 100644 --- a/QC.Doc/find_doc_files.py +++ b/QC.Doc/find_doc_files.py @@ -7,6 +7,7 @@ # Types of files to look for FILE_TYPES = [ + 'all', 'README', 'CODE_OF_CONDUCT', 'CONTRIBUTING' @@ -80,7 +81,9 @@ def find_file(file_type, extensions, check_single_path=None): def main(): args = get_input_args() - if not args.file_type: + + FILE_TYPES.remove('all') + if not args.file_type or args.file_type in ['all']: args.file_type = FILE_TYPES else: args.file_type = [args.file_type] diff --git a/tooling.json b/tooling.json index 5edd279..6d8408f 100644 --- a/tooling.json +++ b/tooling.json @@ -321,7 +321,7 @@ "args": [ { "type": "optional", "description": "Type of file to look for in the code repository", "option": "--file-type", "value": [ - "README", "CODE_OF_CONDUCT", "CONTRIBUTING" + "all", "README", "CODE_OF_CONDUCT", "CONTRIBUTING" ], "format": "array", "selectable": true, "repeatable": false }, { From b12a7320a83c4d5e905126856b2ea5bfe05a3076 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Fri, 20 May 2022 08:44:07 +0200 Subject: [PATCH 40/63] is_semver.py -> get_git_tags.py --- tooling.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tooling.json b/tooling.json index 6d8408f..494abe2 100644 --- a/tooling.json +++ b/tooling.json @@ -339,8 +339,8 @@ } }, "Semantic Versioning": { - "is_semver.py": { - "docs": "QC.Ver/is_semver.py", + "get_git_tags.py": { + "docs": "QC.Ver/get_git_tags.py", "docker": { "dockerfile": "QC.Ver/Dockerfile", "reviewed": "2022-05-10" @@ -510,7 +510,7 @@ "docs": "https://indigo-dc.github.io/sqa-baseline/#semantic-versioning-qc.ver" }, "tools": { - "Semantic Versioning": ["is_semver.py"] + "Semantic Versioning": ["get_git_tags.py"] } }, "SvcQC.Dep": { From 06edc4d95602eaffb59a6c6172324624f254cf81 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Fri, 20 May 2022 08:47:05 +0200 Subject: [PATCH 41/63] Fix: --file-type -> --file_type --- tooling.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tooling.json b/tooling.json index 494abe2..ed053d0 100644 --- a/tooling.json +++ b/tooling.json @@ -320,7 +320,7 @@ }, "args": [ { - "type": "optional", "description": "Type of file to look for in the code repository", "option": "--file-type", "value": [ + "type": "optional", "description": "Type of file to look for in the code repository", "option": "--file_type", "value": [ "all", "README", "CODE_OF_CONDUCT", "CONTRIBUTING" ], "format": "array", "selectable": true, "repeatable": false }, From 1684e44fbb99e2fa93fe6526b7ca756dd25292c8 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Fri, 20 May 2022 09:29:15 +0200 Subject: [PATCH 42/63] Install git --- QC.Ver/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/QC.Ver/Dockerfile b/QC.Ver/Dockerfile index 325ac4c..1dcef5f 100644 --- a/QC.Ver/Dockerfile +++ b/QC.Ver/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.9-alpine LABEL maintainer="Pablo Orviz " -RUN apk --no-cache add build-base +RUN apk --no-cache add build-base git RUN python3 -m pip install GitPython==3.1.27 COPY get_git_tags.py /usr/bin RUN chmod +x /usr/bin/get_git_tags.py From bca823156249a1006b527a76dda33b44d3e6698e Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Fri, 20 May 2022 09:29:31 +0200 Subject: [PATCH 43/63] Return the exact match for HEAD --- QC.Ver/get_git_tags.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/QC.Ver/get_git_tags.py b/QC.Ver/get_git_tags.py index bf192d8..fd8f7e7 100644 --- a/QC.Ver/get_git_tags.py +++ b/QC.Ver/get_git_tags.py @@ -22,8 +22,13 @@ def main(): args = get_input_args() repo = git.Repo(args.repo_path) - tags = repo.git.tag('-l', '--contains', 'HEAD') - tag_list = tags.split('\n') + try: + tags = repo.git.describe('--exact-match', 'HEAD') + tag_list = tags.split('\n') + except git.exc.GitCommandError as e: + tag_list = [] + + return tag_list print(main()) From 8c55395bec3cf80f7a2ea17065ea201660b3a903 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Tue, 24 May 2022 09:01:19 +0200 Subject: [PATCH 44/63] Tools' version added --- tooling.json | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tooling.json b/tooling.json index ed053d0..2947410 100644 --- a/tooling.json +++ b/tooling.json @@ -2,6 +2,7 @@ "tools": { "scm": { "git": { + "version": "2.34.1", "docs": "https://git-scm.com/", "docker": { "image": "bitnami/git:2.34.1", @@ -24,6 +25,7 @@ }, "Dockerfile": { "hadolint": { + "version": "v2.8.0-0-g398770f-dirty", "docs": "https://github.com/hadolint/hadolint", "docker": { "image": "hadolint/hadolint:2.8.0-alpine", @@ -39,6 +41,7 @@ }, "Java": { "checkstyle": { + "version": "8.43", "docs": "https://github.com/checkstyle/checkstyle", "docker": { "image": "davrodgon/checkstyle@sha256:2ed1d4efd817afde40b219057935139818ee6b88105e75010bde53cd65786bf2", @@ -51,6 +54,7 @@ }, "JSON": { "jsonlint": { + "version": "1.6.3", "docs": "https://github.com/zaach/jsonlint", "docker": { "dockerfile": "QC.Sty/jsonlint/Dockerfile", @@ -64,6 +68,7 @@ }, "Python": { "bandit": { + "version": "1.7.4", "docs": "https://bandit.readthedocs.io/", "docker": { "dockerfile": "QC.Sec/bandit/Dockerfile", @@ -95,6 +100,7 @@ } }, "flake8": { + "version": "4.0.1 (mccabe: 0.6.1, pycodestyle: 2.8.0, pyflakes: 2.4.0)", "docs": "https://flake8.pycqa.org/", "docker": { "image": "pipelinecomponents/flake8:0.9.0", @@ -111,6 +117,7 @@ } }, "pycodestyle": { + "version": "2.8.0", "docs": "https://pycodestyle.pycqa.org/", "docker": { "dockerfile": "QC.Sty/pycodestyle/Dockerfile", @@ -122,6 +129,7 @@ }, "tox": { "jepl_support": true, + "version": "3.14.0", "docs": "https://tox.readthedocs.io/", "docker": { "image": "painless/tox@sha256:4f31b329507f9ab658e3c36c5252ad063cabe3d978d6836e0ba322ccb79af060", @@ -140,6 +148,7 @@ "fair": { "fair-eva": { "executable" : "fair-eva.py", + "version": "latest", "docs": "QC.FAIR/fair-eva.py", "docker": { "dockerfile": "QC.FAIR/Dockerfile", @@ -158,6 +167,7 @@ } }, "F-UJI": { + "version": "1.3.8", "docs": "https://github.com/pangaea-data-publisher/fuji/blob/master/README.md", "docker": { "image": "steinsteiny/fuji:v1.3.8.full", @@ -176,6 +186,7 @@ }, "license": { "licensee": { + "version": "9.15.1", "docs": "https://github.com/licensee/licensee", "docker": { "dockerfile": "QC.Lic/licensee/Dockerfile", @@ -200,6 +211,7 @@ } }, "checkLicense": { + "version": "latest", "docs": "https://github.com/EOSC-synergy/sqa-composer-templates/tree/main/QC.Lic", "docker": { "dockerfile": "Qc.Lic/Dockerfile", @@ -210,6 +222,7 @@ }, "Citation File Format": { "cff-converter-python": { + "version": "2.0.0", "docs": "https://github.com/citation-file-format/cff-converter-python", "docker": { "dockerfile": "QC.Met/cff-converter-python/Dockerfile", @@ -230,6 +243,7 @@ }, "CodeMeta": { "checkCitable": { + "version": "latest", "docs": "https://github.com/EOSC-synergy/sqa-composer-templates/tree/main/QC.Met", "docker": { "dockerfile": "Qc.Met/checkCitable/Dockerfile", @@ -238,6 +252,7 @@ } }, "validate_codemeta": { + "version": "latest", "docs": "", "docker": { "image": "alpine:3.15.0", @@ -258,6 +273,7 @@ }, "Markdown": { "markdownlint": { + "version": "0.11.0", "docs": "https://github.com/markdownlint/markdownlint", "docker": { "dockerfile": "QC.Doc/markdownlint/Dockerfile", @@ -282,6 +298,7 @@ }, "reStructuredText": { "restructuredtext-lint": { + "version": "1.3.2", "docs": "https://github.com/twolfson/restructuredtext-lint", "docker": { "dockerfile": "QC.Doc/restructuredtext-lint/Dockerfile", @@ -313,6 +330,7 @@ }, "Contribution-enabling Files": { "find_doc_files.py": { + "version": "latest", "docs": "QC.Doc/find_doc_files.py", "docker": { "dockerfile": "QC.Doc/Dockerfile", @@ -340,6 +358,7 @@ }, "Semantic Versioning": { "get_git_tags.py": { + "version": "latest", "docs": "QC.Ver/get_git_tags.py", "docker": { "dockerfile": "QC.Ver/Dockerfile", @@ -353,6 +372,7 @@ }, "Kubernetes": { "kubectl (via kustomize)": { + "version": "1.23.6", "docs": "https://kubernetes.io/docs/tasks/manage-kubernetes-objects/kustomization/", "docker": { "image": "bitnami/kubectl:1.23", @@ -375,6 +395,7 @@ ] }, "kubectl (via configuration files)": { + "version": "1.23.6", "docs": "https://kubernetes.io/docs/tasks/manage-kubernetes-objects/declarative-config/", "docker": { "image": "bitnami/kubectl:1.23", From 7b9155d3b02cb46a406dae28487aa1b33fd9a2ea Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Mon, 30 May 2022 13:18:04 +0200 Subject: [PATCH 45/63] Add tools for IM & EC3 --- tooling.json | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tooling.json b/tooling.json index 2947410..03cd124 100644 --- a/tooling.json +++ b/tooling.json @@ -412,6 +412,36 @@ ] } }, + "Infrastructure Manager": { + "im_client": { + "version": "1.5.11", + "docs": "https://imdocs.readthedocs.io/", + "docker": { + "image": "grycap/im-client:ec3", + "reviewed": "2022-05-26" + }, + "template": "im_client", + "args": [ + { + "type": "positional", "description": "Path to deployment file (RADL or TOSCA)", "id": "im_config_files", "value": "", "format": "string", "selectable": true, "repeatable": false + } + ] + }, + "ec3_client": { + "version": "1.5.11", + "docs": "https://ec3.readthedocs.io/en/latest/", + "docker": { + "image": "grycap/im-client:ec3", + "reviewed": "2022-05-26" + }, + "template": "ec3_client", + "args": [ + { + "type": "positional", "description": "Path to deployment file (RADL or TOSCA)", "id": "im_config_files", "value": "", "format": "string", "selectable": true, "repeatable": false + } + ] + } + }, "default": { "commands": { "docs": "https://indigo-dc.github.io/jenkins-pipeline-library/2.0.0/user/config_file.html#commands", @@ -545,6 +575,10 @@ "Kubernetes": [ "kubectl (via kustomize)", "kubectl (via configuration files)" + ], + "Infrastructure Manager": [ + "im_client", + "ec3_client" ] } }, From 6fae110746baa00cb7a734190820e323298946bc Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Mon, 30 May 2022 17:23:41 +0200 Subject: [PATCH 46/63] Add arg for choosing the OpenStack site in IM deployment --- tooling.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tooling.json b/tooling.json index 03cd124..dceb071 100644 --- a/tooling.json +++ b/tooling.json @@ -424,6 +424,9 @@ "args": [ { "type": "positional", "description": "Path to deployment file (RADL or TOSCA)", "id": "im_config_files", "value": "", "format": "string", "selectable": true, "repeatable": false + }, + { + "type": "positional", "description": "OpenStack infrastructure to use for the deployment", "value": ["incd"], "format": "array", "selectable": true, "repeatable": false } ] }, From 794f505d9cefe23cfbdada4c35de7bc533f02653 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Mon, 30 May 2022 17:43:17 +0200 Subject: [PATCH 47/63] Set entrypoint for im_client --- tooling.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tooling.json b/tooling.json index dceb071..70d7a6a 100644 --- a/tooling.json +++ b/tooling.json @@ -418,7 +418,9 @@ "docs": "https://imdocs.readthedocs.io/", "docker": { "image": "grycap/im-client:ec3", - "reviewed": "2022-05-26" + "reviewed": "2022-05-26", + "oneshot": false, + "entrypoint": "[/bin/sleep, infinity]" }, "template": "im_client", "args": [ From 55ad30951216e181d85a9da2ea3809897809c13c Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Mon, 30 May 2022 18:25:40 +0200 Subject: [PATCH 48/63] Set environment --- tooling.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tooling.json b/tooling.json index 70d7a6a..6c536c2 100644 --- a/tooling.json +++ b/tooling.json @@ -420,7 +420,8 @@ "image": "grycap/im-client:ec3", "reviewed": "2022-05-26", "oneshot": false, - "entrypoint": "[/bin/sleep, infinity]" + "entrypoint": "[/bin/sleep, infinity]", + "environment": ["OPENSTACK_USER", "OPENSTACK_PASS", "IM_USER", "IM_PASS"] }, "template": "im_client", "args": [ From 1f710b29d4d9d8da58e88695ddec5f5af27a420f Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Tue, 31 May 2022 12:43:02 +0200 Subject: [PATCH 49/63] im_config_files -> im_config_file --- tooling.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tooling.json b/tooling.json index 6c536c2..8851840 100644 --- a/tooling.json +++ b/tooling.json @@ -426,7 +426,7 @@ "template": "im_client", "args": [ { - "type": "positional", "description": "Path to deployment file (RADL or TOSCA)", "id": "im_config_files", "value": "", "format": "string", "selectable": true, "repeatable": false + "type": "positional", "description": "Path to deployment file (RADL or TOSCA)", "id": "im_config_file", "value": "", "format": "string", "selectable": true, "repeatable": false }, { "type": "positional", "description": "OpenStack infrastructure to use for the deployment", "value": ["incd"], "format": "array", "selectable": true, "repeatable": false From a53e8693685b3931a2ec903ee62c88687bc69b9b Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Tue, 31 May 2022 13:17:12 +0200 Subject: [PATCH 50/63] Add id for openstack_site_id --- tooling.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tooling.json b/tooling.json index 8851840..2262291 100644 --- a/tooling.json +++ b/tooling.json @@ -429,7 +429,7 @@ "type": "positional", "description": "Path to deployment file (RADL or TOSCA)", "id": "im_config_file", "value": "", "format": "string", "selectable": true, "repeatable": false }, { - "type": "positional", "description": "OpenStack infrastructure to use for the deployment", "value": ["incd"], "format": "array", "selectable": true, "repeatable": false + "type": "positional", "description": "OpenStack infrastructure to use for the deployment", "id": "openstack_site_id", "value": ["incd"], "format": "array", "selectable": true, "repeatable": false } ] }, From 4e4584ee3544ab3eebb04c42c0c694feb43725e6 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Wed, 1 Jun 2022 09:49:33 +0200 Subject: [PATCH 51/63] Fix: turn to list --- tooling.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tooling.json b/tooling.json index 2262291..90a3237 100644 --- a/tooling.json +++ b/tooling.json @@ -420,7 +420,7 @@ "image": "grycap/im-client:ec3", "reviewed": "2022-05-26", "oneshot": false, - "entrypoint": "[/bin/sleep, infinity]", + "entrypoint": ["/bin/sleep", "infinity"], "environment": ["OPENSTACK_USER", "OPENSTACK_PASS", "IM_USER", "IM_PASS"] }, "template": "im_client", From 627c177b3814bd53edfcf5da5653d9b18c855d6a Mon Sep 17 00:00:00 2001 From: wilkos-dans Date: Thu, 2 Jun 2022 10:06:49 +0200 Subject: [PATCH 52/63] Add FUJI executer. --- QC.FAIR/fuji.py | 60 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 QC.FAIR/fuji.py diff --git a/QC.FAIR/fuji.py b/QC.FAIR/fuji.py new file mode 100644 index 0000000..cd24e06 --- /dev/null +++ b/QC.FAIR/fuji.py @@ -0,0 +1,60 @@ +#!/usr/bin/python3 + +import argparse +import json +import requests + + +def get_input_args(): + parser = argparse.ArgumentParser(description=( + 'Prepare requests for FAIR assessment tool' + )) + parser.add_argument( + '-OID', + metavar='object_identifier', + type=str, + help='Object Persistent Identifier. I.e.: DOI, Handle, LandingPage ' + 'URL, etc.', + required=True + ) + parser.add_argument( + '--tool_endpoint', + metavar='tool_endpoint', + type=str, + help='Tool endpoint to perform HTTP request. Example: ' + 'http://localhost:1071/fuji/api/v1/evaluate', + required=True + ) + parser.add_argument( + '-T', + metavar='metadata_service_type', + choices=['oai_pmh', 'sparql', 'ogc_csw'], + type=str, + help='Type of the Metadata service. One of: "oai_pmh", "sparql" or ' + '"ogc_csw".' + ) + parser.add_argument( + '-E', + metavar='metadata_service_endpoint', + type=str, + help='Metadata service endpoint URL.' + ) + return parser.parse_args() + + +def main(): + args = get_input_args() + url = args.tool_endpoint + headers = {'Content-Type': 'application/json'} + data = { + "object_identifier": args.OID, + "metadata_service_endpoint": args.E, + "metadata_service_type": args.T, + "test_debug": True, + "use_datacite": True + } + r = requests.post(url, data=json.dumps(data), headers=headers) + return json.dumps(r.text) + + +print(main()) From a11177f8091562fdde0ef067c802ede21b96ef9f Mon Sep 17 00:00:00 2001 From: wilkos-dans Date: Thu, 2 Jun 2022 10:19:52 +0200 Subject: [PATCH 53/63] Updated FAIR FUJI Tool --- tooling.json | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/tooling.json b/tooling.json index 90a3237..aa24460 100644 --- a/tooling.json +++ b/tooling.json @@ -167,21 +167,32 @@ } }, "F-UJI": { - "version": "1.3.8", "docs": "https://github.com/pangaea-data-publisher/fuji/blob/master/README.md", "docker": { - "image": "steinsteiny/fuji:v1.3.8.full", + "image": "steinsteiny/fuji:latest", "oneshot": false, - "reviewed": "2021-07-20" + "reviewed": "2022-06-02" }, - "executable": "curl", + "executable": "fuji.py", "args": [ - {"type": "optional", "description": "Facilitate content negotiation.", "option": "-H", "value": "'Accept: application/json'", "selectable": false}, - {"type": "optional", "description": "", "option": "-H", "value": "\"Content-Type: application/json\"", "selectable": false}, - {"type": "optional", "description": "BasicAuth credentials", "option": "-H", "value": "\"Authorization: Basic bWFydmVsOndvbmRlcndvbWFu\"", "selectable": false}, - {"type": "optional", "description": "HTTP Method", "option": "-X", "value": "POST", "selectable": false}, - {"type": "optional", "description": "Evaluator's input parameters", "option": "-d", "value": "'{\"metadata_service_endpoint\":\"http://digital.csic.es/dspace-oai/request\",\"metadata_service_type\":\"oai_pmh\",\"object_identifier\":\"http://hdl.handle.net/10261/157765\",\"test_debug\":true,\"use_datacite\":true}'", "selectable": false}, - {"type": "positional", "description": "Evaluator's endpoint", "value": "http://localhost:1071/fuji/api/v1/evaluate", "selectable": false}] + { + "type": "optional", "format": "string", "description": "Object Identifier", "option": "-OID", "value": "http://hdl.handle.net/10261/157765", "selectable": true + }, + { + "type": "optional", "format": "string", "description": "F-UJI endpoint", "option": "--tool_endpoint", "value": "http://localhost:1071/fuji/api/v1/evaluate", "selectable": true + }, + { + "type": "optional", "format": "array", "description": "The type of the Metadata Service:", "option": "-T", "value": ["oai_pmh", "sparql", "ogc_csw"], "selectable": true, "repeatable": false + }, + { + "type": "optional", "format": "string", "description": "Metadata Service Endpoint", "option": "-E", "value": "http://digital.csic.es/dspace-oai/request", "selectable": true + } + ], + "reporting": { + "validator": "F-UJI", + "threshold": 0, + "requirement_level": "REQUIRED" + } } }, "license": { From 37cb5ea1fb88b93d6011de0fff91c284d44b541a Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Thu, 2 Jun 2022 14:04:48 +0200 Subject: [PATCH 54/63] Set dummy(boolean) validator for im_client --- tooling.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tooling.json b/tooling.json index aa24460..0d9bf3a 100644 --- a/tooling.json +++ b/tooling.json @@ -442,7 +442,11 @@ { "type": "positional", "description": "OpenStack infrastructure to use for the deployment", "id": "openstack_site_id", "value": ["incd"], "format": "array", "selectable": true, "repeatable": false } - ] + ], + "reporting": { + "validator": "boolean", + "requirement_level": "REQUIRED" + } }, "ec3_client": { "version": "1.5.11", From d4b4ca66d712d9321517500586904fa155b853ff Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Fri, 3 Jun 2022 10:56:10 +0200 Subject: [PATCH 55/63] Add new tools_qaa_specific --- tooling.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tooling.json b/tooling.json index 90a3237..4d8b26f 100644 --- a/tooling.json +++ b/tooling.json @@ -586,6 +586,9 @@ "im_client", "ec3_client" ] + }, + "tools_qaa_specific": { + "scm": ["git"] } }, "SvcQC.Fun": { From 599acd84e71065856161a01e194e76570be168d7 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Fri, 3 Jun 2022 11:53:23 +0200 Subject: [PATCH 56/63] Move git as qaa-specific for QC.Doc --- tooling.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tooling.json b/tooling.json index 34252a9..2b59d3f 100644 --- a/tooling.json +++ b/tooling.json @@ -503,6 +503,9 @@ "Markdown": ["markdownlint"], "reStructuredText": ["restructuredtext-lint"], "Contribution-enabling Files": ["find_doc_files.py"] + }, + "tools_qaa_specific": { + "scm": ["git"] } }, "QC.FAIR": { @@ -601,9 +604,6 @@ "im_client", "ec3_client" ] - }, - "tools_qaa_specific": { - "scm": ["git"] } }, "SvcQC.Fun": { From 1cae913490222188ae3a972ab79061281c974bd2 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Fri, 3 Jun 2022 13:09:20 +0200 Subject: [PATCH 57/63] Fix: F-UJI validator name is 'fuji' --- tooling.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tooling.json b/tooling.json index 2b59d3f..9ce3353 100644 --- a/tooling.json +++ b/tooling.json @@ -189,7 +189,7 @@ } ], "reporting": { - "validator": "F-UJI", + "validator": "fuji", "threshold": 0, "requirement_level": "REQUIRED" } From 27ca3370000d97c166c26db4189ba1bb7910f774 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Thu, 9 Jun 2022 13:18:18 +0200 Subject: [PATCH 58/63] Multiple subcriteria per tool --- tooling.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tooling.json b/tooling.json index 9ce3353..87690de 100644 --- a/tooling.json +++ b/tooling.json @@ -19,7 +19,7 @@ "reporting": { "validator": "boolean", "requirement_level": "REQUIRED", - "subcriterion": "QC.Acc01" + "subcriterion": ["QC.Acc01", "QC.Doc01.1"] } } }, From 147f233e0779d246afbd9d4a4af3f93d2a0bf473 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Fri, 10 Jun 2022 13:45:29 +0200 Subject: [PATCH 59/63] im_client uses 'jenkins_exit_status' as validator --- tooling.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tooling.json b/tooling.json index 87690de..ee69d56 100644 --- a/tooling.json +++ b/tooling.json @@ -444,7 +444,8 @@ } ], "reporting": { - "validator": "boolean", + "validator": "jenkins_exit_status", + "subcriterion": "SvcQC.Dep01", "requirement_level": "REQUIRED" } }, From 62fb3db3cb6cf56299c5856ea728256c0a9afc8e Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Wed, 13 Jul 2022 14:17:13 +0200 Subject: [PATCH 60/63] Remove kubectl --- tooling.json | 41 +++++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/tooling.json b/tooling.json index ee69d56..71143d7 100644 --- a/tooling.json +++ b/tooling.json @@ -458,10 +458,39 @@ }, "template": "ec3_client", "args": [ - { - "type": "positional", "description": "Path to deployment file (RADL or TOSCA)", "id": "im_config_files", "value": "", "format": "string", "selectable": true, "repeatable": false - } - ] + { + "type": "positional", + "description": "Path to EC3 templates in the local repository", + "id": "ec3_templates_repo_dir", + "value": "./", + "format": "string", + "selectable": true, + "repeatable": true + }, + { + "type": "positional", + "description": "EC3 template names used to deploy the cluster (without '.radl' extension)", + "id": "ec3_templates", + "value": "", + "format": "string", + "selectable": true, + "repeatable": true + }, + { + "type": "positional", + "description": "OpenStack infrastructure to use for the deployment", + "id": "openstack_site_id", + "value": ["ifca"], + "format": "array", + "selectable": true, + "repeatable": false + } + ], + "reporting": { + "validator": "jenkins_exit_status", + "subcriterion": "SvcQC.Dep01", + "requirement_level": "REQUIRED" + } } }, "default": { @@ -597,10 +626,6 @@ "docs": "https://eosc-synergy.github.io/service-qa-baseline/#dep" }, "tools": { - "Kubernetes": [ - "kubectl (via kustomize)", - "kubectl (via configuration files)" - ], "Infrastructure Manager": [ "im_client", "ec3_client" From b4317686abe36637bfc99a93b323416b5ab0515b Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Wed, 13 Jul 2022 14:41:24 +0200 Subject: [PATCH 61/63] Disable more than one template dir --- tooling.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tooling.json b/tooling.json index 71143d7..e4e3b75 100644 --- a/tooling.json +++ b/tooling.json @@ -464,7 +464,7 @@ "id": "ec3_templates_repo_dir", "value": "./", "format": "string", - "selectable": true, + "selectable": false, "repeatable": true }, { From ac85baaa7ccc11de51d2639bb3a622c81f1de031 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Wed, 13 Jul 2022 14:43:07 +0200 Subject: [PATCH 62/63] Fix: make it selectable --- tooling.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tooling.json b/tooling.json index e4e3b75..52db98c 100644 --- a/tooling.json +++ b/tooling.json @@ -464,8 +464,8 @@ "id": "ec3_templates_repo_dir", "value": "./", "format": "string", - "selectable": false, - "repeatable": true + "selectable": true, + "repeatable": false }, { "type": "positional", From b360a2111353b69f968580c360bfcc691b1bc094 Mon Sep 17 00:00:00 2001 From: Pablo Orviz Date: Mon, 18 Jul 2022 10:36:31 +0200 Subject: [PATCH 63/63] Add docker entrypoint and environment for ec3-client --- tooling.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tooling.json b/tooling.json index 52db98c..34aa145 100644 --- a/tooling.json +++ b/tooling.json @@ -450,11 +450,14 @@ } }, "ec3_client": { - "version": "1.5.11", + "version": "2.2.1", "docs": "https://ec3.readthedocs.io/en/latest/", "docker": { "image": "grycap/im-client:ec3", - "reviewed": "2022-05-26" + "reviewed": "2022-05-26", + "oneshot": false, + "entrypoint": ["/bin/sleep", "infinity"], + "environment": ["OPENSTACK_USER", "OPENSTACK_PASS", "IM_USER", "IM_PASS"] }, "template": "ec3_client", "args": [