From fa21208d8b214d4e6854bfa09c05e6bde1ddc43b Mon Sep 17 00:00:00 2001 From: CaptainOfHacks <39195263+CaptainOfHacks@users.noreply.github.com> Date: Tue, 16 Jan 2024 20:14:59 +0200 Subject: [PATCH 1/5] update RMLMapper version --- Makefile | 2 +- infra/airflow-cluster/Dockerfile | 2 +- infra/airflow/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 02eab246..0882a9ca 100644 --- a/Makefile +++ b/Makefile @@ -213,7 +213,7 @@ stop-metabase: init-rml-mapper: @ echo -e "RMLMapper folder initialisation!" @ mkdir -p ./.rmlmapper - @ wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v6.1.3/rmlmapper-6.1.3-r367-all.jar -O ./.rmlmapper/rmlmapper.jar + @ wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v6.5.1/rmlmapper-6.5.1-r371-all.jar -O ./.rmlmapper/rmlmapper.jar init-limes: @ echo -e "Limes folder initialisation!" diff --git a/infra/airflow-cluster/Dockerfile b/infra/airflow-cluster/Dockerfile index 954e9550..8d657f67 100644 --- a/infra/airflow-cluster/Dockerfile +++ b/infra/airflow-cluster/Dockerfile @@ -14,7 +14,7 @@ COPY requirements.txt /opt/airflow # working in the /opt/airflow WORKDIR /opt/airflow RUN mkdir -p ./.rmlmapper -RUN wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v6.1.3/rmlmapper-6.1.3-r367-all.jar -O ./.rmlmapper/rmlmapper.jar +RUN wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v6.5.1/rmlmapper-6.5.1-r371-all.jar -O ./.rmlmapper/rmlmapper.jar RUN wget -c https://kumisystems.dl.sourceforge.net/project/saxon/Saxon-HE/10/Java/SaxonHE10-6J.zip -P .saxon/ diff --git a/infra/airflow/Dockerfile b/infra/airflow/Dockerfile index 27d682c0..28f99355 100644 --- a/infra/airflow/Dockerfile +++ b/infra/airflow/Dockerfile @@ -14,7 +14,7 @@ COPY requirements.txt /opt/airflow # working in the /opt/airflow WORKDIR /opt/airflow RUN mkdir -p ./.rmlmapper ./dags ./ted_sws ./temp -RUN wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v6.1.3/rmlmapper-6.1.3-r367-all.jar -O ./.rmlmapper/rmlmapper.jar +RUN wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v6.5.1/rmlmapper-6.5.1-r371-all.jar -O ./.rmlmapper/rmlmapper.jar RUN wget -c https://kumisystems.dl.sourceforge.net/project/saxon/Saxon-HE/10/Java/SaxonHE10-6J.zip -P .saxon/ From 298c1c18c5ef828d5da281bab03bfee9b2e91bfa Mon Sep 17 00:00:00 2001 From: CaptainOfHacks <39195263+CaptainOfHacks@users.noreply.github.com> Date: Wed, 17 Jan 2024 21:16:40 +0200 Subject: [PATCH 2/5] fix inherit dependency updates --- requirements.dev.txt | 2 +- requirements.txt | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements.dev.txt b/requirements.dev.txt index 7e45e5f0..55e7280b 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -7,6 +7,6 @@ pytest-subtests~=0.6.0 tox~=3.24.5 tox-pytest-summary~=0.1.2 mongomock==4.1.2 -uvicorn[standard] +uvicorn[standard]==0.24.0 allure-pytest-bdd==2.10.0 pycurl~=7.45.2 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index aa4cd4bc..cc29684e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,4 +25,5 @@ ordered-set~=4.0.2 json2html~=1.3.0 minio~=7.1.1 certifi~=2022.12.7 -shortuuid~=1.0.11 \ No newline at end of file +shortuuid~=1.0.11 +pendulum~=2.1.2 \ No newline at end of file From 9753f28664dadf2b963022ffb156662a62a84292 Mon Sep 17 00:00:00 2001 From: CaptainOfHacks <39195263+CaptainOfHacks@users.noreply.github.com> Date: Wed, 17 Jan 2024 21:37:16 +0200 Subject: [PATCH 3/5] Revert "update RMLMapper version" This reverts commit fa21208d8b214d4e6854bfa09c05e6bde1ddc43b. --- Makefile | 2 +- infra/airflow-cluster/Dockerfile | 2 +- infra/airflow/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 0882a9ca..02eab246 100644 --- a/Makefile +++ b/Makefile @@ -213,7 +213,7 @@ stop-metabase: init-rml-mapper: @ echo -e "RMLMapper folder initialisation!" @ mkdir -p ./.rmlmapper - @ wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v6.5.1/rmlmapper-6.5.1-r371-all.jar -O ./.rmlmapper/rmlmapper.jar + @ wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v6.1.3/rmlmapper-6.1.3-r367-all.jar -O ./.rmlmapper/rmlmapper.jar init-limes: @ echo -e "Limes folder initialisation!" diff --git a/infra/airflow-cluster/Dockerfile b/infra/airflow-cluster/Dockerfile index 8d657f67..954e9550 100644 --- a/infra/airflow-cluster/Dockerfile +++ b/infra/airflow-cluster/Dockerfile @@ -14,7 +14,7 @@ COPY requirements.txt /opt/airflow # working in the /opt/airflow WORKDIR /opt/airflow RUN mkdir -p ./.rmlmapper -RUN wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v6.5.1/rmlmapper-6.5.1-r371-all.jar -O ./.rmlmapper/rmlmapper.jar +RUN wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v6.1.3/rmlmapper-6.1.3-r367-all.jar -O ./.rmlmapper/rmlmapper.jar RUN wget -c https://kumisystems.dl.sourceforge.net/project/saxon/Saxon-HE/10/Java/SaxonHE10-6J.zip -P .saxon/ diff --git a/infra/airflow/Dockerfile b/infra/airflow/Dockerfile index 28f99355..27d682c0 100644 --- a/infra/airflow/Dockerfile +++ b/infra/airflow/Dockerfile @@ -14,7 +14,7 @@ COPY requirements.txt /opt/airflow # working in the /opt/airflow WORKDIR /opt/airflow RUN mkdir -p ./.rmlmapper ./dags ./ted_sws ./temp -RUN wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v6.5.1/rmlmapper-6.5.1-r371-all.jar -O ./.rmlmapper/rmlmapper.jar +RUN wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v6.1.3/rmlmapper-6.1.3-r367-all.jar -O ./.rmlmapper/rmlmapper.jar RUN wget -c https://kumisystems.dl.sourceforge.net/project/saxon/Saxon-HE/10/Java/SaxonHE10-6J.zip -P .saxon/ From 342decab680c84b7b88f9c71c9ac87877cb9c7f7 Mon Sep 17 00:00:00 2001 From: CaptainOfHacks <39195263+CaptainOfHacks@users.noreply.github.com> Date: Fri, 19 Jan 2024 10:38:37 +0200 Subject: [PATCH 4/5] remove YARRRML2RML converter --- .../adapters/yarrrml2rml_converter.py | 35 ------------------- .../test_yarrrml2rml_converter.py | 20 ----------- 2 files changed, 55 deletions(-) delete mode 100644 ted_sws/mapping_suite_processor/adapters/yarrrml2rml_converter.py delete mode 100644 tests/e2e/mapping_suite_processor/test_yarrrml2rml_converter.py diff --git a/ted_sws/mapping_suite_processor/adapters/yarrrml2rml_converter.py b/ted_sws/mapping_suite_processor/adapters/yarrrml2rml_converter.py deleted file mode 100644 index 5099e108..00000000 --- a/ted_sws/mapping_suite_processor/adapters/yarrrml2rml_converter.py +++ /dev/null @@ -1,35 +0,0 @@ -import abc -import pathlib -import subprocess - - -class YARRRML2RMLConverterABC(abc.ABC): - """ - This class is a general interface of a YARRRML to RML converter. - """ - - @abc.abstractmethod - def convert(self, yarrrml_input_file_path: pathlib.Path, rml_output_file_path: pathlib.Path): - """ - This method converts a yarrrml file and writes the result to another rml file. - :param yarrrml_input_file_path: - :param rml_output_file_path: - :return: - """ - - -class YARRRML2RMLConverter(YARRRML2RMLConverterABC): - """ - This class converts YARRRML to RML using an external docker container that performs conversion logic. - """ - - def convert(self, yarrrml_input_file_path: pathlib.Path, - rml_output_file_path: pathlib.Path) -> subprocess.CompletedProcess: - """ - This method converts a YARRRML file and writes the result to another RML file. - :param yarrrml_input_file_path: - :param rml_output_file_path: - :return: - """ - bash_script = f"(docker run --rm -i -v {yarrrml_input_file_path.parent}:/data rmlio/yarrrml-parser:latest -i /data/{yarrrml_input_file_path.name}) > {rml_output_file_path}" - return subprocess.run(bash_script, shell=True, capture_output=True) diff --git a/tests/e2e/mapping_suite_processor/test_yarrrml2rml_converter.py b/tests/e2e/mapping_suite_processor/test_yarrrml2rml_converter.py deleted file mode 100644 index 35c579fe..00000000 --- a/tests/e2e/mapping_suite_processor/test_yarrrml2rml_converter.py +++ /dev/null @@ -1,20 +0,0 @@ -import os -import pathlib -from tempfile import NamedTemporaryFile - -from ted_sws.mapping_suite_processor.adapters.yarrrml2rml_converter import YARRRML2RMLConverter - - -def test_yarrrml_to_rml_convertor(yarrrml_file_content, rml_file_result): - yarrrml_file = NamedTemporaryFile(mode="w+",suffix=".yaml") - rml_file = NamedTemporaryFile(mode="w+") - yarrrml_file_path = pathlib.Path(yarrrml_file.name) - rml_file_path = pathlib.Path(rml_file.name) - yarrrml_file.write(yarrrml_file_content) - yarrrml_file.seek(0, os.SEEK_SET) - yarrrml2rml_converter = YARRRML2RMLConverter() - yarrrml2rml_converter.convert(yarrrml_input_file_path = yarrrml_file_path, - rml_output_file_path= rml_file_path - ) - rml_result = rml_file.read() - assert rml_result == rml_file_result From a83824a413ae06c26654174938715af85f71b788 Mon Sep 17 00:00:00 2001 From: CaptainOfHacks <39195263+CaptainOfHacks@users.noreply.github.com> Date: Fri, 19 Jan 2024 10:38:55 +0200 Subject: [PATCH 5/5] fix notice publisher test with new METS package name --- tests/features/notice_publisher/test_notice_publisher.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/features/notice_publisher/test_notice_publisher.py b/tests/features/notice_publisher/test_notice_publisher.py index 84bd76ab..9f71e39a 100644 --- a/tests/features/notice_publisher/test_notice_publisher.py +++ b/tests/features/notice_publisher/test_notice_publisher.py @@ -78,7 +78,8 @@ def the_notice_publication_by_id_is_executed(publish_notice_id, notice_repositor def the_mets_package_available_in_a_shared_sftp_drive(published_notice: Notice, sftp_remote_folder_path): """the METS package available in a shared SFTP drive.""" publisher: SFTPPublisher = SFTPPublisher() - remote_notice_path = f"{sftp_remote_folder_path}/{published_notice.ted_id}.zip" + assert published_notice.mets_manifestation + remote_notice_path = f"{sftp_remote_folder_path}/{published_notice.mets_manifestation.package_name}" publisher.connect() assert publisher.exists(remote_path=remote_notice_path) publisher.disconnect()