From 3a09c099771a3b28e2133575a61f407c2f17b6cf Mon Sep 17 00:00:00 2001 From: ammar Date: Wed, 5 Apr 2023 14:30:40 +0200 Subject: [PATCH 1/3] Copy input data to reports folder Signed-off-by: ammar --- keiko/keiko/keiko.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/keiko/keiko/keiko.py b/keiko/keiko/keiko.py index 9c66551545a..e5befb7bc0a 100644 --- a/keiko/keiko/keiko.py +++ b/keiko/keiko/keiko.py @@ -106,13 +106,17 @@ def generate_report( preprocessed_tex = Path(tmp_dirname) / tex_output_file_name preprocessed_tex.write_text(out_document) - # copy preprocessed tex file if debug is enabled + # if debug is enabled copy preprocessed tex file and input data if debug or settings.debug: shutil.copyfile( Path(tmp_dirname) / tex_output_file_name, Path(settings.reports_folder) / tex_output_file_name, ) + Path(settings.reports_folder).joinpath(report_id).with_suffix(".json").write_text( + report_data.json(indent=4) + ) + # run pdflatex cmd = [ "pdflatex", From f3a6dc5328c538792b3140b74cb289dcf0d8fce4 Mon Sep 17 00:00:00 2001 From: ammar Date: Wed, 5 Apr 2023 15:11:40 +0200 Subject: [PATCH 2/3] Some improvements Signed-off-by: ammar --- keiko/keiko/api.py | 3 +-- keiko/keiko/keiko.py | 46 ++++++++++++++++++++--------------------- keiko/keiko/settings.py | 19 ++++++++--------- 3 files changed, 33 insertions(+), 35 deletions(-) diff --git a/keiko/keiko/api.py b/keiko/keiko/api.py index 123cbc86b9a..f8d1352a213 100644 --- a/keiko/keiko/api.py +++ b/keiko/keiko/api.py @@ -58,8 +58,7 @@ def health() -> ServiceHealth: return get_health() # mount reports as static files - if not Path(settings.reports_folder).exists(): - Path(settings.reports_folder).mkdir(parents=True) + Path(settings.reports_folder).mkdir(parents=True, exist_ok=True) app.mount("/reports", StaticFiles(directory=settings.reports_folder), name="reports") return app diff --git a/keiko/keiko/keiko.py b/keiko/keiko/keiko.py index e5befb7bc0a..45ff4266777 100644 --- a/keiko/keiko/keiko.py +++ b/keiko/keiko/keiko.py @@ -87,46 +87,47 @@ def generate_report( logger.info("Template rendered. [report_id=%s] [template=%s]", report_id, template_name) # create temp folder - with tempfile.TemporaryDirectory() as tmp_dirname: + with tempfile.TemporaryDirectory() as directory: logger.info( - "Temporary folder created. [report_id=%s] [template=%s] [tmp_dirname=%s]", + "Temporary folder created. [report_id=%s] [template=%s] [directory=%s]", report_id, template_name, - tmp_dirname, + directory, ) # copy assets - shutil.copytree(settings.assets_folder, tmp_dirname, dirs_exist_ok=True) + shutil.copytree(settings.assets_folder, directory, dirs_exist_ok=True) logger.info("Assets copied. [report_id=%s] [template=%s]", report_id, template_name) + output_file = settings.reports_folder / report_id + # tex file - tex_output_file_name = report_id + ".keiko.tex" - pdf_output_file_name = report_id + ".keiko.pdf" + tex_output_file_path = output_file.with_suffix(".keiko.tex") + pdf_output_file_path = output_file.with_suffix(".keiko.pdf") - preprocessed_tex = Path(tmp_dirname) / tex_output_file_name - preprocessed_tex.write_text(out_document) + preprocessed_tex_path = Path(directory).joinpath(f"{report_id}.keiko.tex") + preprocessed_tex_path.write_text(out_document) # if debug is enabled copy preprocessed tex file and input data if debug or settings.debug: shutil.copyfile( - Path(tmp_dirname) / tex_output_file_name, - Path(settings.reports_folder) / tex_output_file_name, + preprocessed_tex_path, + tex_output_file_path, ) - Path(settings.reports_folder).joinpath(report_id).with_suffix(".json").write_text( - report_data.json(indent=4) - ) + json_output_file_path = output_file.with_suffix(".keiko.json") + json_output_file_path.write_text(report_data.json(indent=4)) # run pdflatex cmd = [ "pdflatex", "-synctex=1", "-interaction=nonstopmode", - tex_output_file_name, + preprocessed_tex_path.as_posix(), ] - env = {**os.environ, "TEXMFVAR": tmp_dirname} + env = {**os.environ, "TEXMFVAR": directory} for i in (1, 2): - output = subprocess.run(cmd, cwd=tmp_dirname, env=env, capture_output=True, check=False) + output = subprocess.run(cmd, cwd=directory, env=env, capture_output=True, check=False) logger.info( "pdflatex [run=%d] [report_id=%s] [template=%s] [command=%s]", i, @@ -143,16 +144,15 @@ def generate_report( logger.debug(output.stderr.decode("utf-8")) # copy result back to output folder - Path(settings.reports_folder).mkdir(parents=True, exist_ok=True) shutil.copyfile( - Path(tmp_dirname) / pdf_output_file_name, - Path(settings.reports_folder) / pdf_output_file_name, + preprocessed_tex_path.with_suffix(".pdf"), + pdf_output_file_path, ) logger.info( "Report copied to reports folder. [report_id=%s] [template=%s] [output_file=%s]", report_id, template_name, - Path(settings.reports_folder) / pdf_output_file_name, + pdf_output_file_path, ) # ...tempfiles are deleted automatically when leaving the context @@ -161,8 +161,8 @@ def generate_report( def read_glossary(glossary: str, settings: Settings) -> Dict[str, Tuple[str, str]]: """Read a glossary CSV file and return a dictionary of entries.""" glossary_entries = {} - glossary_file_path = Path(settings.glossaries_folder) / glossary - with open(glossary_file_path, encoding="utf-8") as glossary_file: + glossary_file_path = settings.glossaries_folder / glossary + with glossary_file_path.open(encoding="utf-8") as glossary_file: csvreader = csv.reader(glossary_file) # skip header _ = next(csvreader) @@ -170,5 +170,5 @@ def read_glossary(glossary: str, settings: Settings) -> Dict[str, Tuple[str, str # only allow words with baretext representation bare_word = baretext(row[0]) if bare_word != "": - glossary_entries[baretext(row[0])] = (row[0], row[1]) + glossary_entries[baretext(row[0])] = row[0], row[1] return glossary_entries diff --git a/keiko/keiko/settings.py b/keiko/keiko/settings.py index 6f90c4d10c4..d19536a295f 100644 --- a/keiko/keiko/settings.py +++ b/keiko/keiko/settings.py @@ -1,18 +1,17 @@ """Keiko settings module.""" -from pydantic import BaseSettings, Field +from pydantic import BaseSettings, Field, DirectoryPath, FilePath class Settings(BaseSettings): """Application settings loaded from environment variables.""" - debug: bool = Field(False, env="KEIKO_DEBUG") - log_cfg: str = Field( - "logging.json", - env="KEIKO_LOG_CFG", - ) + debug: bool = Field(False, description="Enable debug mode") + log_cfg: FilePath = Field("logging.json", description="Path to the logging configuration file") + templates_folder: DirectoryPath = Field("templates", description="Folder containing the templates") + glossaries_folder: DirectoryPath = Field("glossaries", description="Folder containing the glossaries") + assets_folder: DirectoryPath = Field("assets", description="Folder containing the assets") + reports_folder: DirectoryPath = Field("/reports", description="Output folder containing the reports") - templates_folder: str = Field("templates", env="KEIKO_TEMPLATES_FOLDER") - glossaries_folder: str = Field("glossaries", env="KEIKO_GLOSSARIES_FOLDER") - reports_folder: str = Field("/reports", env="KEIKO_REPORTS_FOLDER") - assets_folder: str = Field("assets", env="KEIKO_ASSETS_FOLDER") + class Config: + env_prefix = "KEIKO_" From 32d7858c3b780486d0c3610260da10adc8d8742c Mon Sep 17 00:00:00 2001 From: ammar Date: Wed, 5 Apr 2023 15:16:52 +0200 Subject: [PATCH 3/3] Made reports folder setting validation less strict Signed-off-by: ammar --- keiko/keiko/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/keiko/keiko/settings.py b/keiko/keiko/settings.py index d19536a295f..a190d3bb4a1 100644 --- a/keiko/keiko/settings.py +++ b/keiko/keiko/settings.py @@ -1,4 +1,5 @@ """Keiko settings module.""" +from pathlib import Path from pydantic import BaseSettings, Field, DirectoryPath, FilePath @@ -11,7 +12,7 @@ class Settings(BaseSettings): templates_folder: DirectoryPath = Field("templates", description="Folder containing the templates") glossaries_folder: DirectoryPath = Field("glossaries", description="Folder containing the glossaries") assets_folder: DirectoryPath = Field("assets", description="Folder containing the assets") - reports_folder: DirectoryPath = Field("/reports", description="Output folder containing the reports") + reports_folder: Path = Field("/reports", description="Output folder containing the reports") class Config: env_prefix = "KEIKO_"