From 1ffebb29aa405135ad9bc84bccc023f908832704 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Mond=C3=A9jar=20Rubio?= Date: Fri, 29 Nov 2024 14:56:06 +0100 Subject: [PATCH] Allow to configure HTTP cache directory (#244) --- .markdownlint-cli2.yaml | 2 + README.md | 22 ++++++++- examples/http-cache-dir/docs/README.md | 21 ++++++++ examples/http-cache-dir/mkdocs.yml | 5 ++ examples/http-cache/docs/README.md | 2 + locale/es/README.md | 20 +++++++- locale/es/README.md.po | 35 ++++++++++---- locale/fr/README.md | 21 +++++++- locale/fr/README.md.po | 35 ++++++++++---- pyproject.toml | 2 +- src/mkdocs_include_markdown_plugin/cache.py | 48 +++++++++++++------ src/mkdocs_include_markdown_plugin/config.py | 1 + src/mkdocs_include_markdown_plugin/plugin.py | 15 +++--- src/mkdocs_include_markdown_plugin/process.py | 9 ++-- .../test_cache_integration.py | 45 +++++++++++++---- tests/test_integration/test_examples.py | 7 +-- tests/test_unit/test_cache.py | 26 ++++++---- 17 files changed, 240 insertions(+), 76 deletions(-) create mode 100644 examples/http-cache-dir/docs/README.md create mode 100644 examples/http-cache-dir/mkdocs.yml diff --git a/.markdownlint-cli2.yaml b/.markdownlint-cli2.yaml index 9e94750..21f05ea 100644 --- a/.markdownlint-cli2.yaml +++ b/.markdownlint-cli2.yaml @@ -1,3 +1,5 @@ config: no-duplicate-heading: false no-inline-html: false + emphasis-style: + style: underscore diff --git a/README.md b/README.md index 0182ab2..4db53ef 100644 --- a/README.md +++ b/README.md @@ -101,8 +101,9 @@ plugins: cache: 600 ``` -In order to use this feature, the dependency [platformdirs] must be installed. -You can include it in the installation of the plugin adding the `cache` extra: +In order to use this feature, the dependency [platformdirs] must be installed +or the setting [`cache_dir`](#cache_dir) must be defined. You can include +[platformdirs] in the installation of the plugin adding the `cache` extra: ```txt # requirements.txt @@ -111,6 +112,23 @@ mkdocs-include-markdown-plugin[cache] +#### `cache_dir` + +Directory where cached HTTP requests will be stored. If set, [platformdirs] is not +needed to be installed to use [`cache`](#cache). + +```yaml +plugins: + - include-markdown: + cache: 600 + cache_dir: ./mkdocs-include-markdown-cache +``` + +A _.gitignore_ file will be added to the cache directory if not exists to avoid +committing the cache files. + + + #### `directives` Customize the names of the directives. diff --git a/examples/http-cache-dir/docs/README.md b/examples/http-cache-dir/docs/README.md new file mode 100644 index 0000000..bc4dbd3 --- /dev/null +++ b/examples/http-cache-dir/docs/README.md @@ -0,0 +1,21 @@ +# Header + +{% + include-markdown "https://raw.githubusercontent.com/mondeja/mkdocs-include-markdown-plugin/master/examples/basic/docs/included.md" + start="<--start-->" +%} + +## mkdocs.yml + +``` +{% + include "https://raw.githubusercontent.com/mondeja/mkdocs-include-markdown-plugin/master/examples/basic/mkdocs.yml" +%} +``` + +## From cache + +{% + include-markdown "https://raw.githubusercontent.com/mondeja/mkdocs-include-markdown-plugin/master/examples/basic/docs/included.md" + start="<--start-->" +%} diff --git a/examples/http-cache-dir/mkdocs.yml b/examples/http-cache-dir/mkdocs.yml new file mode 100644 index 0000000..ebbee01 --- /dev/null +++ b/examples/http-cache-dir/mkdocs.yml @@ -0,0 +1,5 @@ +site_name: Foo +plugins: + - include-markdown: + cache: 600 + cache_dir: .mkdocs-include-markdown-cache diff --git a/examples/http-cache/docs/README.md b/examples/http-cache/docs/README.md index 7e8b565..a975143 100644 --- a/examples/http-cache/docs/README.md +++ b/examples/http-cache/docs/README.md @@ -4,6 +4,7 @@ {% include-markdown "https://raw.githubusercontent.com/mondeja/mkdocs-include-markdown-plugin/master/examples/basic/docs/included.md" + start="<--start-->" %} ## mkdocs.yml @@ -18,4 +19,5 @@ {% include-markdown "https://raw.githubusercontent.com/mondeja/mkdocs-include-markdown-plugin/master/examples/basic/docs/included.md" + start="<--start-->" %} diff --git a/locale/es/README.md b/locale/es/README.md index d46a887..d142858 100644 --- a/locale/es/README.md +++ b/locale/es/README.md @@ -88,14 +88,30 @@ plugins: cache: 600 ``` -Para usar esta funcionalidad, la dependencia [platformdirs] debe ser instalada. -Puedes incluirla en la instalación del plugin añadiendo el extra `cache`: +Para poder utilizar esta función, se debe instalar la dependencia [platformdirs] +o definir la configuración [`cache_dir`](#cache_dir). Puedes incluir +[platformdirs] en la instalación del plugin agregando el extra `cache`: ```txt # requirements.txt mkdocs-include-markdown-plugin[cache] ``` +#### `cache_dir` + +Directorio donde se almacenarán las solicitudes HTTP en caché. Si se configura, +no es necesario instalar [platformdirs] para usar [`cache`](#cache). + +```yaml +plugins: + - include-markdown: + cache: 600 + cache_dir: ./mkdocs-include-markdown-cache +``` + +Se agregará un archivo *.gitignore* al directorio de caché si no existe para +evitar confirmar los archivos de caché. + #### `directives` Personaliza los nombres de las directivas. diff --git a/locale/es/README.md.po b/locale/es/README.md.po index 16bcc87..d13ddde 100644 --- a/locale/es/README.md.po +++ b/locale/es/README.md.po @@ -349,15 +349,6 @@ msgid "" msgstr "" "Relativas desde el archivo que las incluye (empezando por `./` o `../`)." -msgid "" -"In order to use this feature, the dependency [platformdirs] must be " -"installed. You can include it in the installation of the plugin adding the " -"`cache` extra:" -msgstr "" -"Para usar esta funcionalidad, la dependencia [platformdirs] debe ser " -"instalada. Puedes incluirla en la instalación del plugin añadiendo el extra " -"`cache`:" - msgid "`opening_tag` and `closing_tag`" msgstr "`opening_tag` y `closing_tag`" @@ -454,3 +445,29 @@ msgstr "" msgid "Customize the names of the directives." msgstr "Personaliza los nombres de las directivas." + +msgid "" +"In order to use this feature, the dependency [platformdirs] must be " +"installed or the setting [`cache_dir`](#cache_dir) must be defined. You can " +"include [platformdirs] in the installation of the plugin adding the `cache` " +"extra:" +msgstr "" +"Para poder utilizar esta función, se debe instalar la dependencia " +"[platformdirs] o definir la configuración [`cache_dir`](#cache_dir). Puedes " +"incluir [platformdirs] en la instalación del plugin agregando el extra " +"`cache`:" + +msgid "" +"Directory where cached HTTP requests will be stored. If set, [platformdirs] " +"is not needed to be installed to use [`cache`](#cache)." +msgstr "" +"Directorio donde se almacenarán las solicitudes HTTP en caché. Si se " +"configura, no es necesario instalar [platformdirs] para usar " +"[`cache`](#cache)." + +msgid "" +"A *.gitignore* file will be added to the cache directory if not exists to " +"avoid committing the cache files." +msgstr "" +"Se agregará un archivo *.gitignore* al directorio de caché si no existe para" +" evitar confirmar los archivos de caché." diff --git a/locale/fr/README.md b/locale/fr/README.md index 3768766..1e611d8 100644 --- a/locale/fr/README.md +++ b/locale/fr/README.md @@ -88,14 +88,31 @@ plugins: ``` Pour utiliser cette fonctionnalité, la dépendance [platformdirs] doit être -installée. Vous pouvez l'inclure dans l'installation du plugin en ajoutant le -supplément `cache`: +installée ou le paramètre [`cache_dir`](#cache_dir) doit être défini. Vous +pouvez inclure [platformdirs] dans l'installation du plugin en ajoutant le +supplément `cache` : ```txt # requirements.txt mkdocs-include-markdown-plugin[cache] ``` +#### `cache_dir` + +Répertoire dans lequel les requêtes HTTP mises en cache seront stockées. Si +défini, [platformdirs] n'a pas besoin d'être installé pour utiliser +[`cache`](#cache). + +```yaml +plugins: + - include-markdown: + cache: 600 + cache_dir: ./mkdocs-include-markdown-cache +``` + +Un fichier *.gitignore* sera ajouté au répertoire de cache s'il n'existe pas pour +éviter de valider les fichiers de cache. + #### `directives` Personnaliser les noms des directives. diff --git a/locale/fr/README.md.po b/locale/fr/README.md.po index be54f39..1b2f8e6 100644 --- a/locale/fr/README.md.po +++ b/locale/fr/README.md.po @@ -348,15 +348,6 @@ msgid "" "Relative from the file that includes them (starting with `./` or `../`)." msgstr "Relatifs du fichiers qui les inclut (commençant par `./` ou `../`)." -msgid "" -"In order to use this feature, the dependency [platformdirs] must be " -"installed. You can include it in the installation of the plugin adding the " -"`cache` extra:" -msgstr "" -"Pour utiliser cette fonctionnalité, la dépendance [platformdirs] doit être " -"installée. Vous pouvez l'inclure dans l'installation du plugin en ajoutant " -"le supplément `cache`:" - msgid "`opening_tag` and `closing_tag`" msgstr "`opening_tag` et `closing_tag`" @@ -453,3 +444,29 @@ msgstr "" msgid "Customize the names of the directives." msgstr "Personnaliser les noms des directives." + +msgid "" +"In order to use this feature, the dependency [platformdirs] must be " +"installed or the setting [`cache_dir`](#cache_dir) must be defined. You can " +"include [platformdirs] in the installation of the plugin adding the `cache` " +"extra:" +msgstr "" +"Pour utiliser cette fonctionnalité, la dépendance [platformdirs] doit être " +"installée ou le paramètre [`cache_dir`](#cache_dir) doit être défini. Vous " +"pouvez inclure [platformdirs] dans l'installation du plugin en ajoutant le " +"supplément `cache` :" + +msgid "" +"Directory where cached HTTP requests will be stored. If set, [platformdirs] " +"is not needed to be installed to use [`cache`](#cache)." +msgstr "" +"Répertoire dans lequel les requêtes HTTP mises en cache seront stockées. Si " +"défini, [platformdirs] n'a pas besoin d'être installé pour utiliser " +"[`cache`](#cache)." + +msgid "" +"A *.gitignore* file will be added to the cache directory if not exists to " +"avoid committing the cache files." +msgstr "" +"Un fichier *.gitignore* sera ajouté au répertoire de cache s'il n'existe pas" +" pour éviter de valider les fichiers de cache." diff --git a/pyproject.toml b/pyproject.toml index c00deb6..865c447 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "mkdocs-include-markdown-plugin" -version = "7.1.1" +version = "7.1.2" description = "Mkdocs Markdown includer plugin." readme = "README.md" license = "Apache-2.0" diff --git a/src/mkdocs_include_markdown_plugin/cache.py b/src/mkdocs_include_markdown_plugin/cache.py index 137c3f5..90d1f49 100644 --- a/src/mkdocs_include_markdown_plugin/cache.py +++ b/src/mkdocs_include_markdown_plugin/cache.py @@ -9,9 +9,6 @@ from importlib.util import find_spec -CACHE_AVAILABLE = find_spec('platformdirs') is not None - - class Cache: """Cache for arbitrary content, one file per entry.""" @@ -64,31 +61,52 @@ def set_(self, url: str, value: str, encoding: str = 'utf-8') -> None: # noqa: def clean(self) -> None: """Clean expired entries from the cache.""" for fname in os.listdir(self.cache_dir): + if fname == '.gitignore': + continue fpath = os.path.join(self.cache_dir, fname) creation_time = self.get_creation_time_from_fpath(fpath) if time.time() > creation_time + self.expiration_seconds: os.remove(fpath) -def get_cache_directory() -> str | None: - """Get the cache directory.""" - if not CACHE_AVAILABLE: +def get_cache_directory(cache_dir: str) -> str | None: + """Get cache directory.""" + if cache_dir: + return cache_dir + + if not is_platformdirs_installed(): return None try: from platformdirs import user_data_dir except ImportError: return None + else: + return user_data_dir('mkdocs-include-markdown-plugin') + + +def initialize_cache(expiration_seconds: int, cache_dir: str) -> Cache | None: + """Initialize a cache instance.""" + cache_directory = get_cache_directory(cache_dir) + + if cache_directory is None: + return None - cache_dir = user_data_dir('mkdocs-include-markdown-plugin') - os.makedirs(cache_dir, exist_ok=True) + os.makedirs(cache_directory, exist_ok=True) - return cache_dir + # Add a `.gitignore` file to prevent the cache directory from being + # included in the repository. This is needed because the cache directory + # can be configured as a relative path with `cache_dir` setting. + gitignore = os.path.join(cache_directory, '.gitignore') + if not os.path.exists(gitignore): + with open(gitignore, 'wb') as f: + f.write(b'*\n') + cache = Cache(cache_directory, expiration_seconds) + cache.clean() + return cache -def initialize_cache(expiration_seconds: int) -> Cache | None: - """Initialize a cache instance.""" - cache_dir = get_cache_directory() - return None if cache_dir is None else Cache( - cache_dir, expiration_seconds, - ) + +def is_platformdirs_installed() -> bool: + """Check if `platformdirs` package is installed without importing it.""" + return find_spec('platformdirs') is not None diff --git a/src/mkdocs_include_markdown_plugin/config.py b/src/mkdocs_include_markdown_plugin/config.py index 7792b65..047d1d1 100644 --- a/src/mkdocs_include_markdown_plugin/config.py +++ b/src/mkdocs_include_markdown_plugin/config.py @@ -24,6 +24,7 @@ class PluginConfig(Config): # noqa: D101 end = Optional(MkType(str)) exclude = ListOfItems(MkType(str), default=[]) cache = MkType(int, default=0) + cache_dir = MkType(str, default='') recursive = MkType(bool, default=True) directives = MkType( dict, diff --git a/src/mkdocs_include_markdown_plugin/plugin.py b/src/mkdocs_include_markdown_plugin/plugin.py index 602cc0c..948d06d 100644 --- a/src/mkdocs_include_markdown_plugin/plugin.py +++ b/src/mkdocs_include_markdown_plugin/plugin.py @@ -31,22 +31,19 @@ class IncludeMarkdownPlugin(BasePlugin[PluginConfig]): def on_config(self, config: MkDocsConfig) -> MkDocsConfig: if self.config.cache > 0: - cache = initialize_cache(self.config.cache) + cache = initialize_cache(self.config.cache, self.config.cache_dir) if cache is None: raise PluginError( - 'The "platformdirs" package is required to use the' - ' "cache" option. Install' - ' mkdocs-include-markdown-plugin with the "cache"' - ' extra to install it.', + 'Either `cache_dir` global setting must be configured or' + ' `platformdirs` package is required to use the' + ' `cache` option. Install mkdocs-include-markdown-plugin' + " with the 'cache' extra to install `platformdirs`.", ) - cache.clean() self._cache = cache if '__default' not in self.config.directives: # pragma: no cover for directive in self.config.directives: - if directive not in { - 'include', 'include-markdown', - }: + if directive not in ('include', 'include-markdown'): raise PluginError( f"Invalid directive name '{directive}' at 'directives'" ' global setting. Valid values are "include" and' diff --git a/src/mkdocs_include_markdown_plugin/process.py b/src/mkdocs_include_markdown_plugin/process.py index 32eb533..4d41abc 100644 --- a/src/mkdocs_include_markdown_plugin/process.py +++ b/src/mkdocs_include_markdown_plugin/process.py @@ -244,12 +244,9 @@ def rewrite_relative_urls( ``source_path`` will still work when inserted into a file at ``destination_path``. """ - def rewrite_url(url: str) -> str: - from urllib.parse import urlparse, urlunparse - - if is_relative_path(url): - return url + from urllib.parse import urlparse, urlunparse + def rewrite_url(url: str) -> str: scheme, netloc, path, params, query, fragment = urlparse(url) # absolute or mail @@ -425,7 +422,7 @@ def increase_headings_offset(markdown: str, offset: int = 0) -> str: def rstrip_trailing_newlines(content: str) -> str: """Removes trailing newlines from a string.""" - while content.endswith('\n') or content.endswith('\r'): + while content.endswith(('\n', '\r')): content = content.rstrip('\r\n') return content diff --git a/tests/test_integration/test_cache_integration.py b/tests/test_integration/test_cache_integration.py index 527c133..9e4754e 100644 --- a/tests/test_integration/test_cache_integration.py +++ b/tests/test_integration/test_cache_integration.py @@ -7,10 +7,10 @@ import mkdocs_include_markdown_plugin.cache from mkdocs_include_markdown_plugin import IncludeMarkdownPlugin from mkdocs_include_markdown_plugin.cache import ( - CACHE_AVAILABLE, Cache, get_cache_directory, initialize_cache, + is_platformdirs_installed, ) from mkdocs_include_markdown_plugin.event import on_page_markdown from testing_helpers import parametrize_directives @@ -46,12 +46,13 @@ def test_page_included_by_url_is_cached( tmp_path, plugin, ): - cache_dir = get_cache_directory() - if not CACHE_AVAILABLE: - assert cache_dir is None - assert initialize_cache(600) is None + if not is_platformdirs_installed(): + assert initialize_cache(600, '') is None return + cache_dir = get_cache_directory('') + os.makedirs(cache_dir, exist_ok=True) + file_path = os.path.join( cache_dir, Cache.generate_unique_key_from_url(url), ) @@ -84,14 +85,40 @@ def test_cache_setting_when_not_available_raises_error(monkeypatch): @dataclass class FakeConfig: cache: int + cache_dir: str + directives: dict[str, str] monkeypatch.setattr( mkdocs_include_markdown_plugin.cache, - 'CACHE_AVAILABLE', - False, + 'is_platformdirs_installed', + lambda: False, ) plugin = IncludeMarkdownPlugin() - plugin.config = FakeConfig(cache=600) + plugin.config = FakeConfig( + cache=600, cache_dir='', directives={'__default': ''}, + ) with pytest.raises(PluginError) as exc: plugin.on_config({}) - assert 'The "platformdirs" package is required' in str(exc.value) + assert ( + 'Either `cache_dir` global setting must be configured or' + ' `platformdirs` package is required' + ) in str(exc.value) + + +def test_cache_setting_available_with_cache_dir(monkeypatch): + @dataclass + class FakeConfig: + cache: int + cache_dir: str + directives: dict[str, str] + + monkeypatch.setattr( + mkdocs_include_markdown_plugin.cache, + 'is_platformdirs_installed', + lambda: False, + ) + plugin = IncludeMarkdownPlugin() + plugin.config = FakeConfig( + cache=600, cache_dir='foo', directives={'__default': ''}, + ) + plugin.on_config({}) diff --git a/tests/test_integration/test_examples.py b/tests/test_integration/test_examples.py index 4054f4b..147a339 100644 --- a/tests/test_integration/test_examples.py +++ b/tests/test_integration/test_examples.py @@ -7,7 +7,7 @@ from mkdocs.commands.build import build from mkdocs.exceptions import Abort -from mkdocs_include_markdown_plugin.cache import CACHE_AVAILABLE +from mkdocs_include_markdown_plugin.cache import is_platformdirs_installed from testing_helpers import rootdir @@ -25,7 +25,7 @@ def test_examples_subprocess(dirname): config_file = os.path.join(example_dir, 'mkdocs.yml') expected_returncode = 1 if config_is_using_cache_setting( config_file, - ) and not CACHE_AVAILABLE else 0 + ) and not is_platformdirs_installed() else 0 proc = subprocess.Popen( [sys.executable, '-mmkdocs', 'build'], @@ -45,7 +45,8 @@ def test_examples_api(dirname): example_dir = os.path.join(EXAMPLES_DIR, dirname) config_file = os.path.join(example_dir, 'mkdocs.yml') expected_to_raise_exc = ( - config_is_using_cache_setting(config_file) and not CACHE_AVAILABLE + config_is_using_cache_setting(config_file) and + not is_platformdirs_installed() ) def run(): diff --git a/tests/test_unit/test_cache.py b/tests/test_unit/test_cache.py index f578ad3..112ab8e 100644 --- a/tests/test_unit/test_cache.py +++ b/tests/test_unit/test_cache.py @@ -2,10 +2,10 @@ import time from mkdocs_include_markdown_plugin.cache import ( - CACHE_AVAILABLE, Cache, get_cache_directory, initialize_cache, + is_platformdirs_installed, ) @@ -38,15 +38,23 @@ def test_cache_clean(tmp_path): assert len(os.listdir(tmp_path)) == 0 -def test_get_cache_directory(): - if not CACHE_AVAILABLE: - assert get_cache_directory() is None +def test_get_cache_directory_empty(): + if not is_platformdirs_installed(): + assert get_cache_directory('') is None else: - assert isinstance(get_cache_directory(), str) + assert isinstance(get_cache_directory(''), str) -def test_initialize_cache_instance(): - if not CACHE_AVAILABLE: - assert initialize_cache(300) is None +def test_get_cache_directory_custom(): + assert get_cache_directory('foo') == 'foo' + + +def test_initialize_cache_not_cache_dir(): + if not is_platformdirs_installed(): + assert initialize_cache(300, '') is None else: - assert isinstance(initialize_cache(300), Cache) + assert isinstance(initialize_cache(300, ''), Cache) + + +def test_initialize_cache_cache_dir(): + assert isinstance(initialize_cache(300, 'foo'), Cache)