feature: Create digest file to upload

Author: sfauvel

Diff for: mithril-aggregator/src/artifact_builder/cardano_database.rs

@@ -122,7 +122,7 @@ fn compute_uncompressed_database_size(path: &Path) -> StdResult<u64> {
 
 #[cfg(test)]
 mod tests {
-    use std::path::PathBuf;
+    use std::{collections::BTreeMap, path::PathBuf};
 
     use mithril_common::{
         digesters::DummyCardanoDbBuilder,
@@ -136,9 +136,8 @@ mod tests {
     use reqwest::Url;
 
     use crate::{
-        artifact_builder::{
-            MockAncillaryFileUploader, MockDigestFileUploader, MockImmutableFilesUploader,
-        },
+        artifact_builder::{MockAncillaryFileUploader, MockImmutableFilesUploader},
+        immutable_file_digest_mapper::MockImmutableFileDigestMapper,
         test_tools::TestLogger,
         DumbSnapshotter,
     };
@@ -231,12 +230,21 @@ mod tests {
             .unwrap()
         };
 
-        let digest_artifact_builder = DigestArtifactBuilder::new(
-            Url::parse("http://aggregator_uri").unwrap(),
-            vec![],
-            TestLogger::stdout(),
-        )
-        .unwrap();
+        let digest_artifact_builder = {
+            let mut immutable_file_digest_mapper = MockImmutableFileDigestMapper::new();
+
+            immutable_file_digest_mapper
+                .expect_get_immutable_file_digest_map()
+                .returning(|| Ok(BTreeMap::new()));
+
+            DigestArtifactBuilder::new(
+                Url::parse("http://aggregator_uri").unwrap(),
+                vec![],
+                Arc::new(immutable_file_digest_mapper),
+                TestLogger::stdout(),
+            )
+            .unwrap()
+        };
 
         let cardano_database_artifact_builder = CardanoDatabaseArtifactBuilder::new(
             test_dir,

Diff for: mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs

@@ -1,15 +1,19 @@
 use std::{
+    fs,
     path::{Path, PathBuf},
     sync::Arc,
 };
 
-use anyhow::anyhow;
+use anyhow::Context;
 use async_trait::async_trait;
-use mithril_common::{entities::DigestLocation, logging::LoggerExtensions, StdResult};
+use mithril_common::{
+    entities::DigestLocation, logging::LoggerExtensions,
+    messages::CardanoDatabaseDigestListItemMessage, StdResult,
+};
 use reqwest::Url;
-use slog::{debug, error, Logger};
+use slog::{error, Logger};
 
-use crate::{file_uploaders::url_sanitizer::sanitize_url_path, snapshotter::OngoingSnapshot};
+use crate::ImmutableFileDigestMapper;
 
 /// The [DigestFileUploader] trait allows identifying uploaders that return locations for digest archive files.
 #[cfg_attr(test, mockall::automock)]
@@ -24,6 +28,9 @@ pub struct DigestArtifactBuilder {
     aggregator_url_prefix: Url,
     /// Uploaders
     uploaders: Vec<Arc<dyn DigestFileUploader>>,
+
+    immutable_file_digest_mapper: Arc<dyn ImmutableFileDigestMapper>,
+
     logger: Logger,
 }
 
@@ -32,27 +39,60 @@ impl DigestArtifactBuilder {
     pub fn new(
         aggregator_url_prefix: Url,
         uploaders: Vec<Arc<dyn DigestFileUploader>>,
+        immutable_file_digest_mapper: Arc<dyn ImmutableFileDigestMapper>,
         logger: Logger,
     ) -> StdResult<Self> {
         Ok(Self {
             aggregator_url_prefix,
             uploaders,
+            immutable_file_digest_mapper,
             logger: logger.new_with_component_name::<Self>(),
         })
     }
 
     pub async fn upload(&self) -> StdResult<Vec<DigestLocation>> {
-        // let snapshot = self.create_digest_archive(beacon)?;
-        // let digest_path = snapshot.get_file_path();
-        let digest_path = Path::new("");
+        let digest_path = self.create_digest_archive().await?;
 
-        self.upload_digest_archive(digest_path).await
+        let locations = self.upload_digest_archive(&digest_path).await;
+        fs::remove_file(&digest_path).with_context(|| {
+            format!(
+                "Could not remove digest archive file: '{}'",
+                digest_path.display()
+            )
+        })?;
+        locations
     }
 
-    async fn create_digest_archive() -> StdResult<OngoingSnapshot> {
-        // get message service::get_cardano_database_digest_list_message
-        // output json (created from message) to file
-        todo!()
+    async fn create_digest_archive(&self) -> StdResult<PathBuf> {
+        let immutable_file_digest_map = self
+            .immutable_file_digest_mapper
+            .get_immutable_file_digest_map()
+            .await?
+            .into_iter()
+            .map(
+                |(immutable_file_name, digest)| CardanoDatabaseDigestListItemMessage {
+                    immutable_file_name,
+                    digest,
+                },
+            )
+            .collect::<Vec<_>>();
+
+        // TODO : change that injecting the path or using snapshotter
+        let digests_file_path = Path::new("/tmp").join("mithril").join("digests.json");
+
+        if let Some(digests_dir) = digests_file_path.parent() {
+            fs::create_dir_all(digests_dir).with_context(|| {
+                format!(
+                    "Can not create digests directory: '{}'",
+                    digests_dir.display()
+                )
+            })?;
+        }
+
+        let digest_file = fs::File::create(digests_file_path.clone()).unwrap();
+        serde_json::to_writer(digest_file, &immutable_file_digest_map)?;
+
+        Ok(digests_file_path)
     }
 
     /// Uploads the digest archive and returns the locations of the uploaded files.
@@ -94,8 +134,16 @@ impl DigestArtifactBuilder {
 
 #[cfg(test)]
 mod tests {
-    use crate::test_tools::TestLogger;
-    use mithril_common::test_utils::{assert_equivalent, TempDir};
+    use std::{collections::BTreeMap, fs::read_to_string};
+
+    use crate::{
+        immutable_file_digest_mapper::MockImmutableFileDigestMapper, test_tools::TestLogger,
+    };
+    use anyhow::anyhow;
+    use mithril_common::{
+        messages::{CardanoDatabaseDigestListItemMessage, CardanoDatabaseDigestListMessage},
+        test_utils::{assert_equivalent, TempDir},
+    };
 
     use super::*;
 
@@ -121,9 +169,15 @@ mod tests {
 
     #[tokio::test]
     async fn digest_artifact_builder_return_digests_route_on_aggregator() {
+        let mut immutable_file_digest_mapper = MockImmutableFileDigestMapper::new();
+        immutable_file_digest_mapper
+            .expect_get_immutable_file_digest_map()
+            .returning(|| Ok(BTreeMap::new()));
+
         let builder = DigestArtifactBuilder::new(
             Url::parse("https://aggregator/").unwrap(),
             vec![],
+            Arc::new(immutable_file_digest_mapper),
             TestLogger::stdout(),
         )
         .unwrap();
@@ -151,11 +205,14 @@ mod tests {
             let builder = DigestArtifactBuilder::new(
                 Url::parse("https://aggregator/").unwrap(),
                 vec![Arc::new(uploader)],
+                Arc::new(MockImmutableFileDigestMapper::new()),
                 TestLogger::file(&log_path),
             )
             .unwrap();
 
-            let _ = builder.upload_digest_archive(&Path::new("")).await;
+            let _ = builder
+                .upload_digest_archive(Path::new("digest_file"))
+                .await;
         }
 
         let logs = std::fs::read_to_string(&log_path).unwrap();
@@ -169,11 +226,15 @@ mod tests {
         let builder = DigestArtifactBuilder::new(
             Url::parse("https://aggregator/").unwrap(),
             vec![Arc::new(uploader)],
+            Arc::new(MockImmutableFileDigestMapper::new()),
             TestLogger::stdout(),
         )
         .unwrap();
 
-        let locations = builder.upload_digest_archive(&Path::new("")).await.unwrap();
+        let locations = builder
+            .upload_digest_archive(Path::new("digest_file"))
+            .await
+            .unwrap();
 
         assert!(!locations.is_empty());
     }
@@ -193,11 +254,15 @@ mod tests {
         let builder = DigestArtifactBuilder::new(
             Url::parse("https://aggregator/").unwrap(),
             uploaders,
+            Arc::new(MockImmutableFileDigestMapper::new()),
             TestLogger::stdout(),
         )
         .unwrap();
 
-        let locations = builder.upload_digest_archive(&Path::new("")).await.unwrap();
+        let locations = builder
+            .upload_digest_archive(Path::new("digest_file"))
+            .await
+            .unwrap();
 
         assert_equivalent(
             locations,
@@ -223,11 +288,15 @@ mod tests {
         let builder = DigestArtifactBuilder::new(
             Url::parse("https://aggregator/").unwrap(),
             uploaders,
+            Arc::new(MockImmutableFileDigestMapper::new()),
             TestLogger::stdout(),
         )
         .unwrap();
 
-        let locations = builder.upload_digest_archive(&Path::new("")).await.unwrap();
+        let locations = builder
+            .upload_digest_archive(Path::new("digest_file"))
+            .await
+            .unwrap();
 
         assert_equivalent(
             locations,
@@ -244,4 +313,71 @@ mod tests {
             ],
         );
     }
+
+    #[tokio::test]
+    async fn create_digest_archive_should_create_json_file_with_all_digests() {
+        let mut immutable_file_digest_mapper = MockImmutableFileDigestMapper::new();
+        immutable_file_digest_mapper
+            .expect_get_immutable_file_digest_map()
+            .returning(|| {
+                Ok(BTreeMap::from([(
+                    "06685.chunk".to_string(),
+                    "0af556ab2620dd9363bf76963a231abe8948a500ea6be31b131d87907ab09b1e".to_string(),
+                )]))
+            });
+
+        let builder = DigestArtifactBuilder::new(
+            Url::parse("https://aggregator/").unwrap(),
+            vec![],
+            Arc::new(immutable_file_digest_mapper),
+            TestLogger::stdout(),
+        )
+        .unwrap();
+
+        let archive_path = builder.create_digest_archive().await.unwrap();
+        let file_content = read_to_string(archive_path).unwrap();
+        let digest_content: CardanoDatabaseDigestListMessage =
+            serde_json::from_str(&file_content).unwrap();
+
+        assert_eq!(
+            digest_content,
+            vec![CardanoDatabaseDigestListItemMessage {
+                immutable_file_name: "06685.chunk".to_string(),
+                digest: "0af556ab2620dd9363bf76963a231abe8948a500ea6be31b131d87907ab09b1e"
+                    .to_string(),
+            }]
+        );
+    }
+
+    #[tokio::test]
+    async fn upload_should_call_upload_with_created_digest_file_and_delete_the_file() {
+        // TODO : This test is flaky because we create and remove a file with an hard coded path
+        let mut immutable_file_digest_mapper = MockImmutableFileDigestMapper::new();
+        immutable_file_digest_mapper
+            .expect_get_immutable_file_digest_map()
+            .returning(|| Ok(BTreeMap::new()));
+
+        let mut digest_file_uploader = MockDigestFileUploader::new();
+        digest_file_uploader
+            .expect_upload()
+            .withf(|path| path == Path::new("/tmp/mithril/digests.json") && path.exists())
+            .times(1)
+            .return_once(|_| {
+                Ok(DigestLocation::CloudStorage {
+                    uri: "an_uri".to_string(),
+                })
+            });
+
+        let builder = DigestArtifactBuilder::new(
+            Url::parse("https://aggregator/").unwrap(),
+            vec![Arc::new(digest_file_uploader)],
+            Arc::new(immutable_file_digest_mapper),
+            TestLogger::stdout(),
+        )
+        .unwrap();
+
+        let _locations = builder.upload().await.unwrap();
+
+        assert!(!Path::new("/tmp/mithril/digests.json").exists());
+    }
 }

Diff for: mithril-aggregator/src/dependency_injection/builder.rs

@@ -1230,6 +1230,7 @@ impl DependenciesBuilder {
         logger: &Logger,
         cardano_node_version: Version,
         snapshotter: Arc<dyn Snapshotter>,
+        immutable_file_digest_mapper: Arc<dyn ImmutableFileDigestMapper>,
     ) -> Result<CardanoDatabaseArtifactBuilder> {
         let artifacts_dir = Path::new("cardano-database").join("ancillary");
         let snapshot_dir = self
@@ -1265,6 +1266,7 @@ impl DependenciesBuilder {
         let digest_builder = Arc::new(DigestArtifactBuilder::new(
             self.get_server_url_prefix()?,
             vec![],
+            immutable_file_digest_mapper,
             logger.clone(),
         )?);
 
@@ -1305,11 +1307,13 @@ impl DependenciesBuilder {
         let stake_store = self.get_stake_store().await?;
         let cardano_stake_distribution_artifact_builder =
             Arc::new(CardanoStakeDistributionArtifactBuilder::new(stake_store));
+        let immutable_file_digest_mapper = self.get_immutable_file_digest_mapper().await?;
         let cardano_database_artifact_builder =
             Arc::new(self.create_cardano_database_artifact_builder(
                 &logger,
                 cardano_node_version,
                 snapshotter,
+                immutable_file_digest_mapper,
             )?);
         let dependencies = SignedEntityServiceArtifactsDependencies::new(
             mithril_stake_distribution_artifact_builder,
@@ -1825,7 +1829,9 @@ impl DependenciesBuilder {
 mod tests {
     use mithril_common::{entities::SignedEntityTypeDiscriminants, test_utils::TempDir};
 
-    use crate::test_tools::TestLogger;
+    use crate::{
+        immutable_file_digest_mapper::MockImmutableFileDigestMapper, test_tools::TestLogger,
+    };
 
     use super::*;
 
@@ -1888,11 +1894,14 @@ mod tests {
 
         assert!(!ancillary_dir.exists());
 
+        let immutable_file_digest_mapper = MockImmutableFileDigestMapper::new();
+
         dep_builder
             .create_cardano_database_artifact_builder(
                 &TestLogger::stdout(),
                 Version::parse("1.0.0").unwrap(),
                 Arc::new(DumbSnapshotter::new()),
+                Arc::new(immutable_file_digest_mapper),
             )
             .unwrap();