feat: track documentation size in database & metric

migrations/20241018031600_documentation_size.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE builds DROP COLUMN documentation_size;
+ALTER TABLE releases DROP COLUMN source_size;

migrations/20241018031600_documentation_size.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE builds ADD COLUMN documentation_size BIGINT;
+ALTER TABLE releases ADD COLUMN source_size BIGINT;

src/db/add_package.rs

@@ -26,7 +26,7 @@ use tracing::{debug, info, instrument};
 /// NOTE: `source_files` refers to the files originally in the crate,
 /// not the files generated by rustdoc.
 #[allow(clippy::too_many_arguments)]
-#[instrument(skip(conn))]
+#[instrument(skip(conn, compression_algorithms))]
 pub(crate) async fn add_package_into_database(
     conn: &mut sqlx::PgConnection,
     metadata_pkg: &MetadataPackage,
@@ -37,9 +37,10 @@ pub(crate) async fn add_package_into_database(
     registry_data: &ReleaseData,
     has_docs: bool,
     has_examples: bool,
-    compression_algorithms: std::collections::HashSet<CompressionAlgorithm>,
+    compression_algorithms: impl IntoIterator<Item = CompressionAlgorithm>,
     repository_id: Option<i32>,
     archive_storage: bool,
+    source_size: u64,
 ) -> Result<i32> {
     debug!("Adding package into database");
     let crate_id = initialize_crate(conn, &metadata_pkg.name).await?;
@@ -58,12 +59,12 @@ pub(crate) async fn add_package_into_database(
             keywords, have_examples, downloads, files,
             doc_targets, is_library,
             documentation_url, default_target, features,
-            repository_id, archive_storage
+            repository_id, archive_storage, source_size
          )
          VALUES (
             $1,  $2,  $3,  $4,  $5,  $6,  $7,  $8,  $9,
             $10, $11, $12, $13, $14, $15, $16, $17, $18,
-            $19, $20, $21, $22, $23, $24, $25
+            $19, $20, $21, $22, $23, $24, $25, $26
          )
          ON CONFLICT (crate_id, version) DO UPDATE
             SET release_time = $3,
@@ -88,7 +89,8 @@ pub(crate) async fn add_package_into_database(
                 default_target = $22,
                 features = $23,
                 repository_id = $24,
-                archive_storage = $25
+                archive_storage = $25,
+                source_size = $26
          RETURNING id",
         crate_id,
         &metadata_pkg.version,
@@ -114,7 +116,8 @@ pub(crate) async fn add_package_into_database(
         default_target,
         features as Vec<Feature>,
         repository_id,
-        archive_storage
+        archive_storage,
+        source_size as i64,
     )
     .fetch_one(&mut *conn)
     .await?;
@@ -239,6 +242,7 @@ pub(crate) async fn finish_build(
     rustc_version: &str,
     docsrs_version: &str,
     build_status: BuildStatus,
+    documentation_size: Option<u64>,
     errors: Option<&str>,
 ) -> Result<()> {
     debug!("updating build after finishing");
@@ -252,15 +256,17 @@ pub(crate) async fn finish_build(
              build_status = $3,
              build_server = $4,
              errors = $5,
+             documentation_size = $6,
              build_time = NOW()
          WHERE
-            id = $6
+            id = $7
          RETURNING rid",
         rustc_version,
         docsrs_version,
         build_status as BuildStatus,
         hostname.to_str().unwrap_or(""),
         errors,
+        documentation_size.map(|v| v as i64),
         build_id,
     )
     .fetch_one(&mut *conn)
@@ -654,6 +660,7 @@ mod test {
                 "rustc_version",
                 "docsrs_version",
                 BuildStatus::Success,
+                Some(42),
                 None,
             )
             .await?;
@@ -663,6 +670,7 @@ mod test {
                 rustc_version,
                 docsrs_version,
                 build_status as "build_status: BuildStatus",
+                documentation_size,
                 errors
                 FROM builds
                 WHERE id = $1"#,
@@ -674,6 +682,7 @@ mod test {
             assert_eq!(row.rustc_version, Some("rustc_version".into()));
             assert_eq!(row.docsrs_version, Some("docsrs_version".into()));
             assert_eq!(row.build_status, BuildStatus::Success);
+            assert_eq!(row.documentation_size, Some(42));
             assert!(row.errors.is_none());
 
             Ok(())
@@ -694,6 +703,7 @@ mod test {
                 "rustc_version",
                 "docsrs_version",
                 BuildStatus::Failure,
+                None,
                 Some("error message"),
             )
             .await?;
@@ -703,6 +713,7 @@ mod test {
                 rustc_version,
                 docsrs_version,
                 build_status as "build_status: BuildStatus",
+                documentation_size,
                 errors
                 FROM builds
                 WHERE id = $1"#,
@@ -715,6 +726,7 @@ mod test {
             assert_eq!(row.docsrs_version, Some("docsrs_version".into()));
             assert_eq!(row.build_status, BuildStatus::Failure);
             assert_eq!(row.errors, Some("error message".into()));
+            assert!(row.documentation_size.is_none());
 
             Ok(())
         })