Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Leverage manifest file to improve startup speed. #22

Merged
merged 32 commits into from
Sep 10, 2024
Merged
Changes from 1 commit
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
b24626c
Begin initial work at utilizing Django's manifest file for file detec…
Archmonger Aug 30, 2024
007876c
simplify middleware init
Archmonger Sep 3, 2024
201b64c
broken draft implementation
Archmonger Sep 3, 2024
dbba752
Semi-functional solution
Archmonger Sep 6, 2024
3c93ade
Always use asyncio.to_thread
Archmonger Sep 6, 2024
cd47850
Merge branch 'main' into use-django-manifest
Archmonger Sep 6, 2024
d667d9a
drop python 3.8 support
Archmonger Sep 6, 2024
08a4050
non-functional index file serving
Archmonger Sep 7, 2024
02a7cbb
Remove unneeded code block
Archmonger Sep 7, 2024
1429b77
Merge branch 'main' into use-django-manifest
Archmonger Sep 9, 2024
f38dd3f
fix tests
Archmonger Sep 9, 2024
f52dff9
reorder manifest defaults
Archmonger Sep 9, 2024
df751cf
concatenate settings.debug
Archmonger Sep 9, 2024
82b9801
remove unneeded comment
Archmonger Sep 9, 2024
98edffe
no relative imports
Archmonger Sep 9, 2024
4997335
Add no manifest test variants
Archmonger Sep 9, 2024
a2c9496
add changelog
Archmonger Sep 9, 2024
9a9965a
Fix manifest autorefresh
Archmonger Sep 9, 2024
77d3909
Remove useless stat from add_files_from_finders
Archmonger Sep 9, 2024
7bd460c
insert_directory generic
Archmonger Sep 9, 2024
4d22a38
Use stored stat cache
Archmonger Sep 10, 2024
99ab52a
Change `use_finders` behavior
Archmonger Sep 10, 2024
3b5539d
Generic stat_files function
Archmonger Sep 10, 2024
dda35ca
new use_finders docs
Archmonger Sep 10, 2024
f271296
run stat on all postprocessed files
Archmonger Sep 10, 2024
6916d66
remove path resolver
Archmonger Sep 10, 2024
8a45192
Fix legacy Django support
Archmonger Sep 10, 2024
8a9f168
Django 3.2 compatibility
Archmonger Sep 10, 2024
28f0dfb
self-review
Archmonger Sep 10, 2024
49f871b
fix tests failure due to spelling
Archmonger Sep 10, 2024
9e62147
add docs for use_manifest
Archmonger Sep 10, 2024
ab8bb89
Less busy changelog
Archmonger Sep 10, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Generic stat_files function
Archmonger committed Sep 10, 2024
commit 3b5539d8ca4ff96f6a02930c8df54f02d0e320ed
4 changes: 3 additions & 1 deletion src/servestatic/middleware.py
Original file line number Diff line number Diff line change
@@ -23,6 +23,7 @@
AsyncToSyncIterator,
EmptyAsyncIterator,
ensure_leading_trailing_slash,
stat_files,
)
from servestatic.wsgi import ServeStatic

@@ -162,8 +163,9 @@ def add_files_from_finders(self):
files.setdefault(url, storage.path(path))
self.insert_directory(storage.location, self.static_prefix)

stat_cache = stat_files(files.values())
for url, path in files.items():
self.add_file_to_dictionary(url, path)
self.add_file_to_dictionary(url, path, stat_cache=stat_cache)

def add_files_from_manifest(self):
if not isinstance(staticfiles_storage, ManifestStaticFilesStorage):
12 changes: 2 additions & 10 deletions src/servestatic/storage.py
Original file line number Diff line number Diff line change
@@ -16,6 +16,7 @@
from django.core.files.base import ContentFile

from servestatic.compress import Compressor
from servestatic.utils import stat_files

_PostProcessT = Iterator[Union[Tuple[str, str, bool], Tuple[str, None, RuntimeError]]]

@@ -99,7 +100,7 @@ def save_manifest(self):
"paths": self.hashed_files,
"version": self.manifest_version,
"hash": self.manifest_hash,
"stats": self.stat_files(self.hashed_files.keys()),
"stats": stat_files(self.hashed_files.keys(), path_resolver=self.path),
}
if self.manifest_storage.exists(self.manifest_name):
self.manifest_storage.delete(self.manifest_name)
@@ -123,15 +124,6 @@ def load_manifest_stats(self):
f"Couldn't load manifest '{self.manifest_name}' (version {self.manifest_version})"
)

def stat_files(self, relative_paths) -> dict:
"""Stat all files in `relative_paths` concurrently."""
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = {
rel_path: executor.submit(os.stat, self.path(rel_path))
for rel_path in relative_paths
}
return {rel_path: future.result() for rel_path, future in futures.items()}

def post_process_with_compression(self, files):
# Files may get hashed multiple times, we want to keep track of all the
# intermediate files generated during the process and which of these
11 changes: 11 additions & 0 deletions src/servestatic/utils.py
Original file line number Diff line number Diff line change
@@ -32,6 +32,17 @@ def scantree(root):
yield entry.path, entry.stat()


def stat_files(paths, path_resolver=lambda path: path) -> dict:
"""Stat all files in `relative_paths` via threads."""

with concurrent.futures.ThreadPoolExecutor() as executor:
futures = {
rel_path: executor.submit(os.stat, path_resolver(rel_path))
for rel_path in paths
}
return {rel_path: future.result() for rel_path, future in futures.items()}


class AsyncToSyncIterator:
"""Converts any async iterator to sync as efficiently as possible while retaining
full compatibility with any environment.