From b528b02a80b5c2483998dea27598ff6901f9b84d Mon Sep 17 00:00:00 2001 From: Mohamed Feddad Date: Wed, 15 Dec 2021 23:21:42 +0400 Subject: [PATCH 1/6] Add caching module --- flask_minify/cache.py | 44 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 flask_minify/cache.py diff --git a/flask_minify/cache.py b/flask_minify/cache.py new file mode 100644 index 0000000..2deac3d --- /dev/null +++ b/flask_minify/cache.py @@ -0,0 +1,44 @@ +from flask_minify.utils import get_optimized_hashing + + +class MemoryCache: + def __init__(self, store_key_getter=None, limit=0): + self.store_key_getter = store_key_getter + self.limit = limit + self._cache = {} + self.hashing = get_optimized_hashing() + + @property + def store(self): + if self.store_key_getter: + return self._cache.setdefault(self.store_key_getter(), {}) + + return self._cache + + @property + def limit_exceeded(self): + return len(self.store) >= self.limit + + def __getitem__(self, key): + return self.store.get(key) + + def __setitem__(self, key, value): + if self.limit_exceeded: + self.store.popitem() + + self.store.update({key: value}) + + def get_or_set(self, key, getter): + if self.limit == 0: + return getter() + + hashed_key = self.hashing(key.encode("utf-8")).hexdigest() + + if not self[hashed_key]: + self[hashed_key] = getter() + + return self[hashed_key] + + def clear(self): + del self._cache + self._cache = {} From ed862ba934364b9fe75c0a4376481c6806bcebdf Mon Sep 17 00:00:00 2001 From: Mohamed Feddad Date: Wed, 15 Dec 2021 23:22:36 +0400 Subject: [PATCH 2/6] Add caching module tests --- tests/units.py | 57 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/tests/units.py b/tests/units.py index 7c50198..0d59e0a 100644 --- a/tests/units.py +++ b/tests/units.py @@ -1,7 +1,8 @@ +from random import randint from unittest import mock from flask_minify import minify, parsers -from flask_minify.utils import is_cssless, is_empty, is_html, is_js +from flask_minify.cache import MemoryCache from .constants import ( COMPILED_LESS_RAW, @@ -95,3 +96,57 @@ class CustomParser(parsers.Lesscpy): minified = parser.minify(LESS_RAW, "style") assert minified == COMPILED_LESS_RAW + + +class TestMemoryCache: + def setup(self): + self.store_key_getter = lambda: "testing" + self.limit = 2 + self.content = "test something to cache with" + self.to_cache = "testingsomethintocachehopefully" + + def get_cache(self): + return MemoryCache( + self.store_key_getter, + self.limit, + ) + + def test_caching_is_fixed_size(self): + cache = self.get_cache() + getter = lambda: self.to_cache + minified = {cache.get_or_set(self.content, getter) for _ in range(100)} + + assert minified == {self.to_cache} + assert len(cache.store) == 1 + + def test_caching_limit_exceeded(self): + self.limit = 99 + cache = self.get_cache() + getter = lambda: f"{self.to_cache}{randint(1, 999999)}" + scope = 100 + minified = { + cache.get_or_set( + f"{self.content}{i}", + getter, + ) + for i in range(scope) + } + + assert len(minified) == scope + assert len(cache.store) == self.limit + + def test_disable_caching(self): + self.limit = 0 + cache = self.get_cache() + getter = lambda: f"{self.to_cache}{randint(1, 999999)}" + scope = 100 + minified = { + cache.get_or_set( + f"{self.content}{i}", + getter, + ) + for i in range(scope) + } + + assert len(minified) == scope + assert cache.store == {} From 08d35df3bfb6641d67c449b6aed6bfe8868774d8 Mon Sep 17 00:00:00 2001 From: Mohamed Feddad Date: Wed, 15 Dec 2021 23:23:20 +0400 Subject: [PATCH 3/6] Refactor code to use caching module and cleaner utils --- flask_minify/decorators.py | 28 ++++----- flask_minify/main.py | 115 +++++++++++++++---------------------- flask_minify/utils.py | 46 +++------------ 3 files changed, 65 insertions(+), 124 deletions(-) diff --git a/flask_minify/decorators.py b/flask_minify/decorators.py index 54675a6..a3ac3c8 100644 --- a/flask_minify/decorators.py +++ b/flask_minify/decorators.py @@ -1,7 +1,7 @@ from functools import wraps +from flask_minify.cache import MemoryCache from flask_minify.parsers import Parser -from flask_minify.utils import get_optimized_hashing def minify( @@ -9,6 +9,7 @@ def minify( js=False, cssless=False, cache=True, + caching_limit=2, fail_safe=True, parsers={}, ): @@ -24,6 +25,8 @@ def minify( enable minifying CSS/LESS content. cache: bool enable caching minifed response. + caching_limit: int + to limit the number of minified response variations. failsafe: bool silence encountered exceptions. parsers: dict @@ -33,30 +36,21 @@ def minify( ------- String of minified HTML content. """ - hashing = get_optimized_hashing() + caching = MemoryCache(caching_limit if cache else 0) parser = Parser(parsers, fail_safe) parser.update_runtime_options(html, js, cssless) def decorator(function): @wraps(function) def wrapper(*args, **kwargs): - text = function(*args, **kwargs) - key = None - cache_key, cached = function.__dict__.get("minify", (None, None)) - should_minify = isinstance(text, str) and any([html, js, cssless]) + content = function(*args, **kwargs) + should_minify = isinstance(content, str) and any([html, js, cssless]) + get_minified = lambda: parser.minify(content, "html") - if should_minify: - if cache: - key = hashing(text).hexdigest() + if not should_minify: + return content - if cache_key != key or not cache: - text = parser.minify(text, "html") - - if cache: - function.__dict__["minify"] = (key, text) - - should_return_cached = cache_key == key and cache and should_minify - return cached if should_return_cached else text + return caching.get_or_set(content, get_minified) return wrapper diff --git a/flask_minify/main.py b/flask_minify/main.py index da41474..9dba299 100644 --- a/flask_minify/main.py +++ b/flask_minify/main.py @@ -3,8 +3,9 @@ from flask import _app_ctx_stack, request +from flask_minify.cache import MemoryCache from flask_minify.parsers import Parser -from flask_minify.utils import get_optimized_hashing, is_cssless, is_html, is_js +from flask_minify.utils import does_content_type_match class Minify: @@ -84,41 +85,30 @@ def root(id): self.fail_safe = fail_safe self.bypass = bypass self.bypass_caching = bypass_caching - self.caching_limit = caching_limit - self.cache = {} self._app = app self.passive = passive self.static = static - self.hashing = get_optimized_hashing() + self.cache = MemoryCache(self.get_endpoint, caching_limit) self.parser = Parser(parsers, fail_safe) self.parser.update_runtime_options(html, js, cssless, script_types) app and self.init_app(app) - @staticmethod - def get_endpoint_matches(endpoint, patterns): - """Get the patterns that matches the current endpoint. - - Parameters - ---------- - endpoint: str - to finds the matches for. - patterns: list - regex patterns or strings to match endpoint. + def get_endpoint(self): + """Get the current response endpoint, with a failsafe. Returns ------- - list - patterns that match the current endpoint. + str + the current endpoint. """ - matches, x = tee( - compiled_pattern - for compiled_pattern in (compile_re(p) for p in patterns) - if compiled_pattern.search(endpoint) - ) - has_matches = bool(next(x, 0)) + with self.app.app_context(): + path = getattr(request, "endpoint", "") or "" - return matches, has_matches + if path == "static": + path = getattr(request, "path", "") or "" + + return path @property def app(self): @@ -131,23 +121,6 @@ def app(self): """ return self._app or (_app_ctx_stack.top and _app_ctx_stack.top.app) - @property - def endpoint(self): - """Get the current response endpoint, with a failsafe. - - Returns - ------- - str - the current endpoint. - """ - with self.app.app_context(): - path = getattr(request, "endpoint", "") or "" - - if path == "static": - path = getattr(request, "path", "") or "" - - return path - def init_app(self, app): """Handle initiation of multiple apps NOTE:Factory Method""" app.after_request(self.main) @@ -172,27 +145,34 @@ def get_minified_or_cached(self, content, tag): str stored or restored minifed content. """ + _, bypassed = self.get_endpoint_matches(self.bypass_caching) + get_minified = lambda: self.parser.minify(content, tag) - def _cache_dict(): - return self.cache.get(self.endpoint, {}) + if bypassed: + return get_minified() - key = self.hashing(content.encode("utf-8")).hexdigest() - limit_reached = len(_cache_dict()) >= self.caching_limit - _, bypassed = self.get_endpoint_matches(self.endpoint, self.bypass_caching) + return self.cache.get_or_set(content, get_minified) - def _cached(): - return _cache_dict().get(key) - - def _minified(): - return self.parser.minify(content, tag) + def get_endpoint_matches(self, patterns): + """Get the patterns that matches the current endpoint. - if not _cached() and not bypassed: - if limit_reached and _cache_dict(): - _cache_dict().popitem() + Parameters + ---------- + patterns: list + regex patterns or strings to match endpoint. - self.cache.setdefault(self.endpoint, {}).update({key: _minified()}) + Returns + ------- + (iterable, bool) + patterns that match the current endpoint, and True if any matches found + """ + endpoint = self.get_endpoint() + matches, duplicates = tee( + p for p in map(compile_re, patterns) if p.search(endpoint) + ) + has_matches = next(duplicates, 0) != 0 - return _cached() or _minified() + return matches, has_matches def main(self, response): """Where a dragon once lived! @@ -207,25 +187,20 @@ def main(self, response): Flask.Response minified flask response if it fits the requirements. """ - html = is_html(response) - cssless = is_cssless(response) - js = is_js(response) - _, bypassed = self.get_endpoint_matches(self.endpoint, self.bypass) + _, bypassed = self.get_endpoint_matches(self.bypass) should_bypass = bypassed or self.passive - should_minify = any( - [ - html and self.html, - cssless and self.cssless, - js and self.js, - ] + html, cssless, js = does_content_type_match(response) + should_minify = ( + (html and self.html) or (cssless and self.cssless) or (js and self.js) ) if should_minify and not should_bypass: - response.direct_passthrough = False - text = response.get_data(as_text=True) - tag = "html" if html else "script" if js else "style" + if html or (self.static and (cssless or js)): + response.direct_passthrough = False + content = response.get_data(as_text=True) + tag = "html" if html else "script" if js else "style" + minified = self.get_minified_or_cached(content, tag) - if html or (self.static and any([cssless, js])): - response.set_data(self.get_minified_or_cached(text, tag)) + response.set_data(minified) return response diff --git a/flask_minify/utils.py b/flask_minify/utils.py index db705eb..be35991 100644 --- a/flask_minify/utils.py +++ b/flask_minify/utils.py @@ -86,8 +86,8 @@ def get_tag_contents(html, tag, script_types): ) -def is_html(response): - """Check if Flask response of HTML content-type. +def does_content_type_match(response): + """Check if Flask response of content-type match HTML, CSS\LESS or JS. Parameters ---------- @@ -95,43 +95,15 @@ def is_html(response): Returns ------- - True if valid False if not. + (bool, bool, bool) + html, cssless, js if content type match. """ - content_type = getattr(response, "content_type", "") + content_type = getattr(response, "content_type", "").lower() + html = "text/html" in content_type + cssless = "css" in content_type or "less" in content_type + js = "javascript" in content_type - return "text/html" in content_type.lower() - - -def is_js(response): - """Check if Flask response of JS content-type. - - Parameters - ---------- - response: Flask response - - Returns - ------- - True if valid False if not. - """ - content_type = getattr(response, "content_type", "") - - return "javascript" in content_type.lower() - - -def is_cssless(response): - """Check if Flask response of Css or Less content-type. - - Parameters - ---------- - response: Flask response - - Returns - ------- - True if valid False if not. - """ - content_type = getattr(response, "content_type", "") - - return "css" in content_type.lower() or "less" in content_type.lower() + return html, cssless, js def get_optimized_hashing(): From ead307da2c9da86910db3c2f88faa4c640bd2d53 Mon Sep 17 00:00:00 2001 From: Mohamed Feddad Date: Wed, 15 Dec 2021 23:23:40 +0400 Subject: [PATCH 4/6] Refactor tests and resolve failures --- tests/integration.py | 26 +++++++++++--------------- tests/units.py | 27 ++++++++++++++++++--------- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/tests/integration.py b/tests/integration.py index 095fcb7..dd69e81 100644 --- a/tests/integration.py +++ b/tests/integration.py @@ -2,7 +2,6 @@ from flask import send_from_directory from pytest import fixture -from xxhash import xxh64 from .constants import ( FALSE_LESS, @@ -25,13 +24,12 @@ @fixture def client(): + store_minify.cache.clear() store_minify.fail_safe = False store_minify.cssless = True store_minify.js = True - store_minify.caching_limit = 0 store_minify.bypass = [] store_minify.bypass_caching = [] - store_minify.cache = {} store_minify.passive = False store_minify.parser.runtime_options["html"]["minify_inline"] = { "script": True, @@ -89,13 +87,14 @@ def test_lesscss_minify(client): def test_minify_cache(client): """testing caching minifed response""" - store_minify.caching_limit = 10 + store_minify.cache.limit = 10 client.get("/cssless") # hit it twice, to get the cached minified response resp = client.get("/cssless").data assert resp == MINIFED_LESS assert ( - MINIFED_LESS.decode("utf-8") in store_minify.cache.get("cssless", {}).values() + MINIFED_LESS.decode("utf-8") + in store_minify.cache._cache.get("cssless", {}).values() ) @@ -117,23 +116,23 @@ def test_fail_safe_false_input(client): def test_caching_limit_only_when_needed(client): """test caching limit without any variations""" - store_minify.caching_limit = 5 + store_minify.cache.limit = 5 store_minify.cssless = True resp = [client.get("/cssless").data for i in range(10)] - assert len(store_minify.cache.get("cssless", {})) == 1 + assert len(store_minify.cache._cache.get("cssless", {})) == 1 for r in resp: assert MINIFED_LESS == r def test_caching_limit_exceeding(client): """test caching limit with multiple variations""" - store_minify.caching_limit = 4 + new_limit = store_minify.cache.limit = 4 resp = [client.get("/js/{}".format(i)).data for i in range(10)] - assert len(store_minify.cache.get("js_addition", {})) == store_minify.caching_limit + assert len(store_minify.cache._cache.get("js_addition", {})) == new_limit - for v in store_minify.cache.get("js_addition", {}).values(): + for v in store_minify.cache._cache.get("js_addition", {}).values(): assert bytes(v.encode("utf-8")) in resp @@ -143,7 +142,7 @@ def test_bypass_caching(client): resp = client.get("/cssless") resp_values = [ bytes("".format(v).encode("utf-8")) - for v in store_minify.cache.get("cssless", {}).values() + for v in store_minify.cache._cache.get("cssless", {}).values() ] assert MINIFED_LESS == resp.data @@ -182,11 +181,8 @@ def test_html_minify_decorated_cache(client): store_minify.passive = True client.get("/html_decorated").data resp = client.get("/html_decorated").data - hash_key = xxh64(HTML).hexdigest() - cache_tuple = html_decorated.__wrapped__.minify assert resp == MINIFED_HTML - assert cache_tuple == (hash_key, resp.decode("utf-8")) def test_javascript_minify_decorated(client): @@ -279,7 +275,7 @@ def test_script_types(client): ] assert client.get("/js_with_type").data == MINIFIED_JS_WITH_TYPE - store_minify.cache = {} + store_minify.cache.clear() store_minify.parser.runtime_options["html"]["script_types"] = [ "testing", "text/javascript", diff --git a/tests/units.py b/tests/units.py index 0d59e0a..5d1c59a 100644 --- a/tests/units.py +++ b/tests/units.py @@ -3,6 +3,7 @@ from flask_minify import minify, parsers from flask_minify.cache import MemoryCache +from flask_minify.utils import does_content_type_match, is_empty from .constants import ( COMPILED_LESS_RAW, @@ -20,20 +21,29 @@ def test_is_empty(self): def test_is_html(self): """Test is_html check is correct""" - assert is_html(mock.Mock("application/json")) is False - assert is_html(mock.Mock(content_type="text/html")) is True + assert does_content_type_match(mock.Mock("application/json"))[0] is False + assert does_content_type_match(mock.Mock(content_type="text/html"))[0] is True def test_is_js(self): """Test is_js check is correct""" - assert is_js(mock.Mock(content_type="text/html")) is False - assert is_js(mock.Mock(content_type="text/javascript")) is True - assert is_js(mock.Mock(content_type="application/javascript")) is True + assert does_content_type_match(mock.Mock(content_type="text/html"))[2] is False + assert ( + does_content_type_match(mock.Mock(content_type="text/javascript"))[2] + is True + ) + assert ( + does_content_type_match(mock.Mock(content_type="application/javascript"))[2] + is True + ) def test_is_cssless(self): """Test is_cssless check is correct""" - assert is_cssless(mock.Mock(content_type="text/javascript")) is False - assert is_cssless(mock.Mock(content_type="text/css")) is True - assert is_cssless(mock.Mock(content_type="text/less")) is True + assert ( + does_content_type_match(mock.Mock(content_type="text/javascript"))[1] + is False + ) + assert does_content_type_match(mock.Mock(content_type="text/css"))[1] is True + assert does_content_type_match(mock.Mock(content_type="text/less"))[1] is True class TestMinifyRequest: @@ -72,7 +82,6 @@ def test_request_falsy_endpoint(self): self.mock_request.endpoint = None self.bypass.append(endpoint) matches, exists = self.minify_defaults.get_endpoint_matches( - self.minify_defaults.endpoint, [endpoint], ) From deb014e0b9e22a544dae55b6056e6da5ef7a4497 Mon Sep 17 00:00:00 2001 From: Mohamed Feddad Date: Wed, 15 Dec 2021 23:24:19 +0400 Subject: [PATCH 5/6] Bump version to 0.36 --- flask_minify/about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flask_minify/about.py b/flask_minify/about.py index a84ec1d..c12cfdc 100644 --- a/flask_minify/about.py +++ b/flask_minify/about.py @@ -1,4 +1,4 @@ -__version__ = "0.35" +__version__ = "0.36" __doc__ = "Flask extension to minify html, css, js and less." __license__ = "MIT" __author__ = "Mohamed Feddad" From 5d5730b6590e153e166a64c4323ba395297a3140 Mon Sep 17 00:00:00 2001 From: Mohamed Feddad Date: Wed, 15 Dec 2021 23:25:44 +0400 Subject: [PATCH 6/6] Add weekly build to ci flow --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8914740..fd8e4ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,9 @@ name: Build -on: [push] +on: + push: + schedule: + # runs a new build every saturday + - cron: 0 0 * * 6 jobs: build: