fetch publisher data from crates.io, start build artifact caching

Cargo.toml

@@ -71,6 +71,7 @@ bzip2 = "0.4.4"
 serde_cbor = "0.11.1"
 getrandom = "0.2.1"
 itertools = { version = "0.10.5", optional = true}
+sysinfo = { version = "0.28.2", default-features = false }
 rusqlite = { version = "0.29.0", features = ["bundled"] }
 moka = { version ="0.11.0", default-features = false, features = ["sync"]}
 

docker-compose.yml

@@ -14,10 +14,16 @@ services:
         expose: ["3000"]
         volumes:
             - "/var/run/docker.sock:/var/run/docker.sock"
-            - ".rustwide-docker:/opt/docsrs/rustwide"
-            - "cratesio-index:/opt/docsrs/prefix/crates.io-index"
+            # we only need one of the two mappings here. 
+            # this will link to your local filesystem, which makes build debugging 
+            # easier
+            - "./ignored/opt-docsrs:/opt/docsrs_ext/"
+            # this uses a docker volume and will be faster, but you need to 
+            # log into the container to debug its contents
+            # - "prefix-ext:/opt/docsrs_ext/"
         environment:
-            DOCSRS_RUSTWIDE_WORKSPACE: /opt/docsrs/rustwide
+            DOCSRS_RUSTWIDE_WORKSPACE: /opt/docsrs_ext/rustwide
+            DOCSRS_PREFIX: /opt/docsrs_ext/prefix
             DOCSRS_DATABASE_URL: postgresql://cratesfyi:password@db
             DOCSRS_STORAGE_BACKEND: s3
             S3_ENDPOINT: http://s3:9000
@@ -93,4 +99,4 @@ services:
 volumes:
     postgres-data: {}
     minio-data: {}
-    cratesio-index: {}
+    prefix-ext: {}

dockerfiles/entrypoint.sh

@@ -2,7 +2,7 @@
 
 set -euv
 
-export DOCSRS_PREFIX=/opt/docsrs/prefix
+export DOCSRS_PREFIX=${DOCSRS_PREFIX:-"/opt/docsrs/prefix"}
 export DOCSRS_DOCKER=true
 export DOCSRS_LOG=${DOCSRS_LOG-"docs-rs,rustwide=info"}
 export PATH="$PATH:/build/target/release"

src/config.rs

@@ -93,8 +93,13 @@ pub struct Config {
     /// same for the `static.docs.rs` distribution
     pub cloudfront_distribution_id_static: Option<String>,
 
+    // some caches do automatic cleanup based on this free disk space goal.
+    // Value shoudl be between 0 and 1.
+    pub(crate) free_disk_space_goal: f32,
+
     // Build params
     pub(crate) build_attempts: u16,
+    pub(crate) use_build_artifact_cache: bool,
     pub(crate) rustwide_workspace: PathBuf,
     pub(crate) inside_docker: bool,
     pub(crate) docker_image: Option<String>,
@@ -129,7 +134,10 @@ impl Config {
         let prefix: PathBuf = require_env("DOCSRS_PREFIX")?;
 
         Ok(Self {
+            free_disk_space_goal: env("DOCSRS_FREE_DISK_SPACE_GOAL", 0.2)?,
+
             build_attempts: env("DOCSRS_BUILD_ATTEMPTS", 5)?,
+            use_build_artifact_cache: env("DOCSRS_USE_BUILD_ARTIFACT_CACHE", true)?,
 
             crates_io_api_call_retries: env("DOCSRS_CRATESIO_API_CALL_RETRIES", 3)?,
 

src/db/add_package.rs

@@ -2,7 +2,7 @@ use crate::{
     db::types::Feature,
     docbuilder::{BuildResult, DocCoverage},
     error::Result,
-    index::api::{CrateData, CrateOwner, ReleaseData},
+    index::api::{CrateData, GithubUser, ReleaseData},
     storage::CompressionAlgorithm,
     utils::MetadataPackage,
     web::crate_details::CrateDetails,
@@ -371,7 +371,7 @@ pub fn update_crate_data_in_database(
 /// Adds owners into database
 fn update_owners_in_database(
     conn: &mut Client,
-    owners: &[CrateOwner],
+    owners: &[GithubUser],
     crate_id: i32,
 ) -> Result<()> {
     // Update any existing owner data since it is mutable and could have changed since last
@@ -562,9 +562,10 @@ mod test {
                 },
             )?;
 
-            let owner1 = CrateOwner {
+            let owner1 = GithubUser {
                 avatar: "avatar".into(),
                 login: "login".into(),
+                ..Default::default()
             };
 
             update_owners_in_database(&mut conn, &[owner1.clone()], crate_id)?;
@@ -600,16 +601,18 @@ mod test {
             // set initial owner details
             update_owners_in_database(
                 &mut conn,
-                &[CrateOwner {
+                &[GithubUser {
                     login: "login".into(),
                     avatar: "avatar".into(),
+                    ..Default::default()
                 }],
                 crate_id,
             )?;
 
-            let updated_owner = CrateOwner {
+            let updated_owner = GithubUser {
                 login: "login".into(),
                 avatar: "avatar2".into(),
+                ..Default::default()
             };
             update_owners_in_database(&mut conn, &[updated_owner.clone()], crate_id)?;
 
@@ -645,17 +648,19 @@ mod test {
             // set initial owner details
             update_owners_in_database(
                 &mut conn,
-                &[CrateOwner {
+                &[GithubUser {
                     login: "login".into(),
                     avatar: "avatar".into(),
+                    ..Default::default()
                 }],
                 crate_id,
             )?;
 
-            let new_owners: Vec<CrateOwner> = (1..5)
-                .map(|i| CrateOwner {
+            let new_owners: Vec<GithubUser> = (1..5)
+                .map(|i| GithubUser {
                     login: format!("login{i}"),
                     avatar: format!("avatar{i}"),
+                    ..Default::default()
                 })
                 .collect();
 

src/db/delete.rs

@@ -187,7 +187,7 @@ fn delete_crate_from_database(conn: &mut Client, name: &str, crate_id: i32) -> R
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::index::api::CrateOwner;
+    use crate::index::api::GithubUser;
     use crate::test::{assert_success, wrapper};
     use postgres::Client;
     use test_case::test_case;
@@ -313,9 +313,10 @@ mod tests {
                 .name("a")
                 .version("1.0.0")
                 .archive_storage(archive_storage)
-                .add_owner(CrateOwner {
+                .add_owner(GithubUser {
                     login: "malicious actor".into(),
                     avatar: "https://example.org/malicious".into(),
+                    ..Default::default()
                 })
                 .create()?;
             assert!(release_exists(&mut db.conn(), v1)?);
@@ -342,9 +343,10 @@ mod tests {
                 .name("a")
                 .version("2.0.0")
                 .archive_storage(archive_storage)
-                .add_owner(CrateOwner {
+                .add_owner(GithubUser {
                     login: "Peter Rabbit".into(),
                     avatar: "https://example.org/peter".into(),
+                    ..Default::default()
                 })
                 .create()?;
             assert!(release_exists(&mut db.conn(), v2)?);