Cache open repositories to reduce IO & move to zlib-ng

Cargo.toml

@@ -9,6 +9,7 @@ license = "WTFPL"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+arc-swap = "1.7"
 anyhow = "1.0"
 askama = "0.12.0"
 axum = { version = "0.7", features = ["macros"] }
@@ -23,9 +24,9 @@ const-hex = "1.12"
 const_format = "0.2"
 comrak = "0.28.0"
 console-subscriber = { version = "0.4", features = ["parking_lot"] }
-flate2 = "1.0"
+flate2 = { version = "1.0", default-features = false, features = ["zlib-ng"] }
 futures = "0.3"
-gix = "0.66"
+gix = { version = "0.66", features = ["fast-sha1", "zlib-ng"] }
 httparse = "1.7"
 humantime = "2.1"
 itertools = "0.13.0"

flake.nix

@@ -20,6 +20,7 @@
           inherit src;
           strictDeps = true;
           buildInputs = pkgs.lib.optionals pkgs.stdenv.isDarwin [ pkgs.libiconv ];
+          nativeBuildInputs = [ pkgs.cmake ];
           LIBCLANG_PATH = "${pkgs.clang.cc.lib}/lib";
           ROCKSDB_LIB_DIR = "${pkgs.rocksdb}/lib";
         };
@@ -95,6 +96,11 @@
                 description = "Timeout for incoming HTTP requests";
                 type = types.str;
               };
+              pkg = mkOption {
+                default = rgit;
+                description = "rgit package to use";
+                type = types.package;
+              };
             };
 
             config = mkIf cfg.enable {

src/git.rs

@@ -3,6 +3,7 @@ use std::{
     collections::{BTreeMap, VecDeque},
     ffi::OsStr,
     fmt::{self, Arguments, Write},
+    io::ErrorKind,
     path::{Path, PathBuf},
     str::FromStr,
     sync::Arc,
@@ -43,6 +44,7 @@ type ReadmeCacheKey = (PathBuf, Option<Arc<str>>);
 pub struct Git {
     commits: Cache<(ObjectId, bool), Arc<Commit>>,
     readme_cache: Cache<ReadmeCacheKey, Option<(ReadmeFormat, Arc<str>)>>,
+    open_repositories: Cache<PathBuf, ThreadSafeRepository>,
     syntax_set: SyntaxSet,
 }
 
@@ -51,11 +53,15 @@ impl Git {
     pub fn new(syntax_set: SyntaxSet) -> Self {
         Self {
             commits: Cache::builder()
-                .time_to_live(Duration::from_secs(10))
+                .time_to_live(Duration::from_secs(30))
                 .max_capacity(100)
                 .build(),
             readme_cache: Cache::builder()
-                .time_to_live(Duration::from_secs(10))
+                .time_to_live(Duration::from_secs(30))
+                .max_capacity(100)
+                .build(),
+            open_repositories: Cache::builder()
+                .time_to_idle(Duration::from_secs(120))
                 .max_capacity(100)
                 .build(),
             syntax_set,
@@ -70,20 +76,24 @@ impl Git {
         repo_path: PathBuf,
         branch: Option<Arc<str>>,
     ) -> Result<Arc<OpenRepository>> {
-        let repo = tokio::task::spawn_blocking({
-            let repo_path = repo_path.clone();
-            move || {
-                gix::open::Options::isolated()
-                    .open_path_as_is(true)
-                    .open(&repo_path)
-            }
-        })
-        .await
-        .context("Failed to join Tokio task")?
-        .map_err(|err| {
-            error!("{}", err);
-            anyhow!("Failed to open repository")
-        })?;
+        let repo = repo_path.clone();
+        let repo = self
+            .open_repositories
+            .try_get_with_by_ref(&repo_path, async move {
+                tokio::task::spawn_blocking(move || {
+                    gix::open::Options::isolated()
+                        .open_path_as_is(true)
+                        .open(&repo)
+                })
+                .await
+                .context("Failed to join Tokio task")
+                .map_err(|e| std::io::Error::new(ErrorKind::Other, e))?
+                .map_err(|err| {
+                    error!("{}", err);
+                    std::io::Error::new(ErrorKind::Other, "Failed to open repository")
+                })
+            })
+            .await?;
 
         Ok(Arc::new(OpenRepository {
             git: self,

src/main.rs

@@ -328,8 +328,8 @@ async fn run_indexer(
 
 #[must_use]
 pub fn build_asset_hash(v: &[u8]) -> Box<str> {
-    let hasher = xxhash_rust::const_xxh3::xxh3_128(v);
-    let out = const_hex::encode(&hasher.to_be_bytes());
+    let hasher = const_xxh3::xxh3_128(v);
+    let out = const_hex::encode(hasher.to_be_bytes());
     Box::from(out)
 }
 

src/methods/filters.rs

@@ -1,8 +1,13 @@
 // sorry clippy, we don't have a choice. askama forces this on us
 #![allow(clippy::unnecessary_wraps, clippy::trivially_copy_pass_by_ref)]
 
-use std::borrow::Borrow;
+use std::{
+    borrow::Borrow,
+    collections::HashMap,
+    sync::{Arc, LazyLock},
+};
 
+use arc_swap::ArcSwap;
 use time::format_description::well_known::Rfc3339;
 
 pub fn format_time(s: impl Borrow<time::OffsetDateTime>) -> Result<String, askama::Error> {
@@ -25,8 +30,28 @@ pub fn hex(s: &[u8]) -> Result<String, askama::Error> {
     Ok(const_hex::encode(s))
 }
 
-pub fn md5(s: &str) -> Result<String, askama::Error> {
-    Ok(const_hex::encode(md5::compute(s).0))
+pub fn gravatar(email: &str) -> Result<&'static str, askama::Error> {
+    static CACHE: LazyLock<ArcSwap<HashMap<&'static str, &'static str>>> =
+        LazyLock::new(|| ArcSwap::new(Arc::new(HashMap::new())));
+
+    if let Some(res) = CACHE.load().get(email).copied() {
+        return Ok(res);
+    }
+
+    let url = format!(
+        "https://www.gravatar.com/avatar/{}",
+        const_hex::encode(md5::compute(email).0)
+    );
+    let key = Box::leak(Box::from(email));
+    let url = url.leak();
+
+    CACHE.rcu(|curr| {
+        let mut r = (**curr).clone();
+        r.insert(key, url);
+        r
+    });
+
+    Ok(url)
 }
 
 #[allow(dead_code)]

templates/repo/macros/refs.html

@@ -14,7 +14,7 @@
         <td><a href="/{{ repo.display() }}/log/?h={{ name }}">{{ name }}</a></td>
         <td><a href="/{{ repo.display() }}/commit/?id={{ commit.get().hash|hex }}">{{ commit.get().summary }}</a></td>
         <td>
-            <img src="https://www.gravatar.com/avatar/{{ commit.get().author.email|md5 }}?s=13&d=retro" width="13" height="13">
+            <img src="{{ commit.get().author.email|gravatar }}" width="13" height="13">
             {{ commit.get().author.name }}
         </td>
         <td>
@@ -44,7 +44,7 @@
         <td><a href="/{{ repo.display() }}/snapshot?h={{ name }}">{{- name -}}.tar.gz</a></td>
         <td>
             {% if let Some(tagger) = tag.get().tagger -%}
-            <img src="https://www.gravatar.com/avatar/{{ tagger.email|md5 }}?s=13&d=retro" width="13" height="13">
+            <img src="{{ tagger.email|gravatar }}" width="13" height="13">
             {{ tagger.name }}
             {%- endif %}
         </td>
@@ -80,7 +80,7 @@
         </td>
         <td><a href="/{{ repo.display() }}/commit/?id={{ commit.hash|hex }}">{{ commit.summary }}</a></td>
         <td>
-            <img src="https://www.gravatar.com/avatar/{{ commit.author.email|md5 }}?s=13&d=retro" width="13" height="13">
+            <img src="{{ commit.author.email|gravatar }}?s=13&d=retro" width="13" height="13">
             {{ commit.author.name }}
         </td>
     </tr>