diff --git a/storage_stats/src/repository.rs b/storage_stats/src/repository.rs index 0990e7ced7b7dca16258f1e5a0eb5e62f3aeca64..cbd6a7d1d194c64f79c09a66daadb10df641180f 100644 --- a/storage_stats/src/repository.rs +++ b/storage_stats/src/repository.rs @@ -88,22 +88,31 @@ async fn read_deb822_file< download: F, client: &'client Client, url: &'url str, -) -> Result<T> { - let bytes = download(client, url) - .await - .with_context(|| format!("Failed to download {}", url))?; - - let file = tokio::task::block_in_place(|| { - rfc822_like::from_bytes(&bytes[..]) - .with_context(|| format!("Failed to parse {}", url)) - })?; +) -> Result<Option<T>> { + match lift_404( + download(client, url) + .await + .with_context(|| format!("Failed to download {}", url)), + )? { + Some(bytes) => { + let file = tokio::task::block_in_place(|| { + rfc822_like::from_bytes(&bytes[..]) + .with_context(|| format!("Failed to parse {}", url)) + })?; + + debug!("Retrieved repo file"); + Ok(Some(file)) + } - debug!("Retrieved repo file"); - Ok(file) + None => Ok(None), + } } impl Repository { - pub async fn read_release(&self, client: &Client) -> Result<RepositoryRelease> { + pub async fn read_release( + &self, + client: &Client, + ) -> Result<Option<RepositoryRelease>> { read_deb822_file(Client::get, client, &format!("{}/Release", &self.url)).await } @@ -111,7 +120,7 @@ impl Repository { &self, client: &Client, component: &str, - ) -> Result<Vec<SourcePackage>> { + ) -> Result<Option<Vec<SourcePackage>>> { read_deb822_file( get_maybe_compressed_file, client, @@ -125,7 +134,7 @@ impl Repository { client: &Client, component: &str, arch: &str, - ) -> Result<Vec<BinaryPackage>> { + ) -> Result<Option<Vec<BinaryPackage>>> { read_deb822_file( get_maybe_compressed_file, client, diff --git a/storage_stats/src/stats.rs b/storage_stats/src/stats.rs index abc05213b1f080fec10b93da7f37b786346d9b97..3231a93c603ec654a9af1517eefccff91336a8c2 100644 --- a/storage_stats/src/stats.rs +++ b/storage_stats/src/stats.rs @@ -10,7 +10,7 @@ use anyhow::{Context, Result}; use futures::{future::try_join_all, stream, try_join, StreamExt, TryStreamExt}; use serde_derive::Serialize; use std::sync::Arc; -use tracing::{debug, info, instrument}; +use tracing::{debug, info, instrument, warn}; async fn scan_source_packages( client: Client, @@ -18,9 +18,23 @@ async fn scan_source_packages( component: String, agg: Arc<StorageUsageAggregator>, ) -> Result<()> { - for pkg in repo.read_sources(&client, &component).await? { - for file in pkg.files { - agg.add(&repo.id, format!("{}/{}", pkg.directory, file.filename), file.size); + match repo.read_sources(&client, &component).await? { + Some(pkgs) => { + for pkg in pkgs { + for file in pkg.files { + agg.add( + &repo.id, + format!("{}/{}", pkg.directory, file.filename), + file.size, + ); + } + } + } + None => { + warn!( + "Repository {}, component {} is missing its Sources file", + repo.id, component + ); } } @@ -34,8 +48,18 @@ async fn scan_binary_packages( arch: String, agg: Arc<StorageUsageAggregator>, ) -> Result<()> { - for pkg in repo.read_packages(&client, &component, &arch).await? { - agg.add(&repo.id, pkg.filename, pkg.size); + match repo.read_packages(&client, &component, &arch).await? { + Some(pkgs) => { + for pkg in pkgs { + agg.add(&repo.id, pkg.filename, pkg.size); + } + } + None => { + warn!( + "Repository {}, component {}, arch {} is missing its Packages file", + repo.id, component, arch + ); + } } Ok(()) @@ -50,7 +74,14 @@ async fn aggregate_basic_stats( ) -> Result<()> { debug!("Scanning repo"); - let release = repo.read_release(client).await?; + let release = match repo.read_release(client).await? { + Some(release) => release, + None => { + warn!("Repository {} is missing its Release file", repo.id); + return Ok(()); + } + }; + let metadata_size: usize = release.files.iter().map(|entry| entry.size).sum(); agg.register_key(repo.id.to_owned(), metadata_size);