Ignore missing files in snapshots

If a snapshot has any missing files, it would make the entire dashboard
collection fail, which isn't really the ideal behavior to have.

https://phabricator.apertis.org/T8593



Signed-off-by: Ryan Gonzalez <ryan.gonzalez@collabora.com>

storage_stats/src/repository.rs

@@ -88,22 +88,31 @@ async fn read_deb822_file<
     download: F,
     client: &'client Client,
     url: &'url str,
-) -> Result<T> {
-    let bytes = download(client, url)
-        .await
-        .with_context(|| format!("Failed to download {}", url))?;
-
-    let file = tokio::task::block_in_place(|| {
-        rfc822_like::from_bytes(&bytes[..])
-            .with_context(|| format!("Failed to parse {}", url))
-    })?;
+) -> Result<Option<T>> {
+    match lift_404(
+        download(client, url)
+            .await
+            .with_context(|| format!("Failed to download {}", url)),
+    )? {
+        Some(bytes) => {
+            let file = tokio::task::block_in_place(|| {
+                rfc822_like::from_bytes(&bytes[..])
+                    .with_context(|| format!("Failed to parse {}", url))
+            })?;
+
+            debug!("Retrieved repo file");
+            Ok(Some(file))
+        }
 
-    debug!("Retrieved repo file");
-    Ok(file)
+        None => Ok(None),
+    }
 }
 
 impl Repository {
-    pub async fn read_release(&self, client: &Client) -> Result<RepositoryRelease> {
+    pub async fn read_release(
+        &self,
+        client: &Client,
+    ) -> Result<Option<RepositoryRelease>> {
         read_deb822_file(Client::get, client, &format!("{}/Release", &self.url)).await
     }
 
@@ -111,7 +120,7 @@ impl Repository {
         &self,
         client: &Client,
         component: &str,
-    ) -> Result<Vec<SourcePackage>> {
+    ) -> Result<Option<Vec<SourcePackage>>> {
         read_deb822_file(
             get_maybe_compressed_file,
             client,
@@ -125,7 +134,7 @@ impl Repository {
         client: &Client,
         component: &str,
         arch: &str,
-    ) -> Result<Vec<BinaryPackage>> {
+    ) -> Result<Option<Vec<BinaryPackage>>> {
         read_deb822_file(
             get_maybe_compressed_file,
             client,

storage_stats/src/stats.rs

@@ -10,7 +10,7 @@ use anyhow::{Context, Result};
 use futures::{future::try_join_all, stream, try_join, StreamExt, TryStreamExt};
 use serde_derive::Serialize;
 use std::sync::Arc;
-use tracing::{debug, info, instrument};
+use tracing::{debug, info, instrument, warn};
 
 async fn scan_source_packages(
     client: Client,
@@ -18,9 +18,23 @@ async fn scan_source_packages(
     component: String,
     agg: Arc<StorageUsageAggregator>,
 ) -> Result<()> {
-    for pkg in repo.read_sources(&client, &component).await? {
-        for file in pkg.files {
-            agg.add(&repo.id, format!("{}/{}", pkg.directory, file.filename), file.size);
+    match repo.read_sources(&client, &component).await? {
+        Some(pkgs) => {
+            for pkg in pkgs {
+                for file in pkg.files {
+                    agg.add(
+                        &repo.id,
+                        format!("{}/{}", pkg.directory, file.filename),
+                        file.size,
+                    );
+                }
+            }
+        }
+        None => {
+            warn!(
+                "Repository {}, component {} is missing its Sources file",
+                repo.id, component
+            );
         }
     }
 
@@ -34,8 +48,18 @@ async fn scan_binary_packages(
     arch: String,
     agg: Arc<StorageUsageAggregator>,
 ) -> Result<()> {
-    for pkg in repo.read_packages(&client, &component, &arch).await? {
-        agg.add(&repo.id, pkg.filename, pkg.size);
+    match repo.read_packages(&client, &component, &arch).await? {
+        Some(pkgs) => {
+            for pkg in pkgs {
+                agg.add(&repo.id, pkg.filename, pkg.size);
+            }
+        }
+        None => {
+            warn!(
+                "Repository {}, component {}, arch {} is missing its Packages file",
+                repo.id, component, arch
+            );
+        }
     }
 
     Ok(())
@@ -50,7 +74,14 @@ async fn aggregate_basic_stats(
 ) -> Result<()> {
     debug!("Scanning repo");
 
-    let release = repo.read_release(client).await?;
+    let release = match repo.read_release(client).await? {
+        Some(release) => release,
+        None => {
+            warn!("Repository {} is missing its Release file", repo.id);
+            return Ok(());
+        }
+    };
+
     let metadata_size: usize = release.files.iter().map(|entry| entry.size).sum();
 
     agg.register_key(repo.id.to_owned(), metadata_size);