diff --git a/Cargo.lock b/Cargo.lock index 3c519079b64..ee49fef20c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3586,7 +3586,7 @@ dependencies = [ [[package]] name = "mithril-aggregator" -version = "0.6.3" +version = "0.6.4" dependencies = [ "anyhow", "async-trait", @@ -3743,7 +3743,7 @@ dependencies = [ [[package]] name = "mithril-common" -version = "0.4.96" +version = "0.4.97" dependencies = [ "anyhow", "async-trait", diff --git a/mithril-aggregator/Cargo.toml b/mithril-aggregator/Cargo.toml index ecd4d10b6be..b1dd921747c 100644 --- a/mithril-aggregator/Cargo.toml +++ b/mithril-aggregator/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-aggregator" -version = "0.6.3" +version = "0.6.4" description = "A Mithril Aggregator server" authors = { workspace = true } edition = { workspace = true } diff --git a/mithril-aggregator/src/artifact_builder/cardano_database.rs b/mithril-aggregator/src/artifact_builder/cardano_database.rs index 1e6f54b24a9..fd3590842c3 100644 --- a/mithril-aggregator/src/artifact_builder/cardano_database.rs +++ b/mithril-aggregator/src/artifact_builder/cardano_database.rs @@ -1,4 +1,7 @@ -use std::path::{Path, PathBuf}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; use anyhow::{anyhow, Context}; use async_trait::async_trait; @@ -12,12 +15,14 @@ use mithril_common::{ StdResult, }; -use crate::artifact_builder::ArtifactBuilder; +use crate::artifact_builder::{AncillaryArtifactBuilder, ArtifactBuilder}; pub struct CardanoDatabaseArtifactBuilder { - db_directory: PathBuf, // TODO: temporary, will be accessed through another dependency instead of direct path. + db_directory: PathBuf, cardano_node_version: Version, compression_algorithm: CompressionAlgorithm, + #[allow(dead_code)] + ancillary_builder: Arc, } impl CardanoDatabaseArtifactBuilder { @@ -25,11 +30,13 @@ impl CardanoDatabaseArtifactBuilder { db_directory: PathBuf, cardano_node_version: &Version, compression_algorithm: CompressionAlgorithm, + ancillary_builder: Arc, ) -> Self { Self { db_directory, cardano_node_version: cardano_node_version.clone(), compression_algorithm, + ancillary_builder, } } } @@ -55,11 +62,17 @@ impl ArtifactBuilder for CardanoDataba })?; let total_db_size_uncompressed = compute_uncompressed_database_size(&self.db_directory)?; + let locations = ArtifactsLocations { + ancillary: vec![], + digest: vec![], + immutables: vec![], + }; + let cardano_database = CardanoDatabaseSnapshot::new( merkle_root.to_string(), beacon, total_db_size_uncompressed, - ArtifactsLocations::default(), // TODO: temporary default locations, will be injected in next PR. + locations, self.compression_algorithm, &self.cardano_node_version, ); @@ -153,6 +166,7 @@ mod tests { test_dir, &Version::parse("1.0.0").unwrap(), CompressionAlgorithm::Zstandard, + Arc::new(AncillaryArtifactBuilder::new(vec![])), ); let beacon = fake_data::beacon(); @@ -177,7 +191,11 @@ mod tests { "merkleroot".to_string(), beacon, expected_total_size, - ArtifactsLocations::default(), + ArtifactsLocations { + ancillary: vec![], + digest: vec![], + immutables: vec![], + }, CompressionAlgorithm::Zstandard, &Version::parse("1.0.0").unwrap(), ); diff --git a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/ancillary.rs b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/ancillary.rs new file mode 100644 index 00000000000..24c3c5f98e5 --- /dev/null +++ b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/ancillary.rs @@ -0,0 +1,110 @@ +#![allow(dead_code)] +use async_trait::async_trait; +use std::{path::Path, sync::Arc}; + +use mithril_common::{entities::AncillaryLocation, StdResult}; + +use crate::{FileUploader, LocalUploader}; + +/// The [AncillaryFileUploader] trait allows identifying uploaders that return locations for ancillary archive files. +#[cfg_attr(test, mockall::automock)] +#[async_trait] +pub trait AncillaryFileUploader: Send + Sync { + /// Uploads the archive at the given filepath and returns the location of the uploaded file. + async fn upload(&self, filepath: &Path) -> StdResult; +} + +#[async_trait] +impl AncillaryFileUploader for LocalUploader { + async fn upload(&self, filepath: &Path) -> StdResult { + let uri = FileUploader::upload(self, filepath).await?.into(); + + Ok(AncillaryLocation::CloudStorage { uri }) + } +} + +/// The [AncillaryArtifactBuilder] creates an ancillary archive from the cardano database directory (including ledger and volatile directories). +/// The archive is uploaded with the provided uploaders. +pub struct AncillaryArtifactBuilder { + uploaders: Vec>, +} + +impl AncillaryArtifactBuilder { + pub fn new(uploaders: Vec>) -> Self { + Self { uploaders } + } + + pub async fn upload_archive(&self, db_directory: &Path) -> StdResult> { + let mut locations = Vec::new(); + for uploader in &self.uploaders { + // TODO: Temporary preparation work, `db_directory` is used as the ancillary archive path for now. + let location = uploader.upload(db_directory).await?; + locations.push(location); + } + + Ok(locations) + } +} + +#[cfg(test)] +mod tests { + use mockall::predicate::eq; + + use super::*; + + #[tokio::test] + async fn upload_archive_should_return_empty_locations_with_no_uploader() { + let builder = AncillaryArtifactBuilder::new(vec![]); + + let locations = builder.upload_archive(Path::new("whatever")).await.unwrap(); + + assert!(locations.is_empty()); + } + + #[tokio::test] + async fn upload_archive_should_return_all_uploaders_returned_locations() { + let mut first_uploader = MockAncillaryFileUploader::new(); + first_uploader + .expect_upload() + .with(eq(Path::new("archive_path"))) + .times(1) + .return_once(|_| { + Ok(AncillaryLocation::CloudStorage { + uri: "an_uri".to_string(), + }) + }); + + let mut second_uploader = MockAncillaryFileUploader::new(); + second_uploader + .expect_upload() + .with(eq(Path::new("archive_path"))) + .times(1) + .return_once(|_| { + Ok(AncillaryLocation::CloudStorage { + uri: "another_uri".to_string(), + }) + }); + + let uploaders: Vec> = + vec![Arc::new(first_uploader), Arc::new(second_uploader)]; + + let builder = AncillaryArtifactBuilder::new(uploaders); + + let locations = builder + .upload_archive(Path::new("archive_path")) + .await + .unwrap(); + + assert_eq!( + locations, + vec![ + AncillaryLocation::CloudStorage { + uri: "an_uri".to_string() + }, + AncillaryLocation::CloudStorage { + uri: "another_uri".to_string() + } + ] + ); + } +} diff --git a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/mod.rs b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/mod.rs new file mode 100644 index 00000000000..0537f92220b --- /dev/null +++ b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/mod.rs @@ -0,0 +1,4 @@ +//! The module is responsible for creating and uploading the archives of the Cardano database artifacts. +mod ancillary; + +pub use ancillary::*; diff --git a/mithril-aggregator/src/artifact_builder/mod.rs b/mithril-aggregator/src/artifact_builder/mod.rs index 45c7facc815..c3aaffadf8d 100644 --- a/mithril-aggregator/src/artifact_builder/mod.rs +++ b/mithril-aggregator/src/artifact_builder/mod.rs @@ -1,5 +1,6 @@ //! The module used for building artifact mod cardano_database; +mod cardano_database_artifacts; mod cardano_immutable_files_full; mod cardano_stake_distribution; mod cardano_transactions; @@ -7,6 +8,7 @@ mod interface; mod mithril_stake_distribution; pub use cardano_database::*; +pub use cardano_database_artifacts::*; pub use cardano_immutable_files_full::*; pub use cardano_stake_distribution::*; pub use cardano_transactions::*; diff --git a/mithril-aggregator/src/configuration.rs b/mithril-aggregator/src/configuration.rs index 7311705b993..94837c41179 100644 --- a/mithril-aggregator/src/configuration.rs +++ b/mithril-aggregator/src/configuration.rs @@ -284,7 +284,7 @@ impl Configuration { .map_err(|e| anyhow!(ConfigError::Message(e.to_string()))) } - /// Return the file of the SQLite stores. If the directory does not exist, it is created. + /// Return the directory of the SQLite stores. If the directory does not exist, it is created. pub fn get_sqlite_dir(&self) -> PathBuf { let store_dir = &self.data_stores_directory; @@ -295,6 +295,15 @@ impl Configuration { self.data_stores_directory.clone() } + /// Return the snapshots directory. + pub fn get_snapshot_dir(&self) -> StdResult { + if !&self.snapshot_directory.exists() { + std::fs::create_dir_all(&self.snapshot_directory)?; + } + + Ok(self.snapshot_directory.clone()) + } + /// Same as the [store retention limit][Configuration::store_retention_limit] but will never /// yield a value lower than 3. /// diff --git a/mithril-aggregator/src/dependency_injection/builder.rs b/mithril-aggregator/src/dependency_injection/builder.rs index bf987fc4674..bc55d2a1999 100644 --- a/mithril-aggregator/src/dependency_injection/builder.rs +++ b/mithril-aggregator/src/dependency_injection/builder.rs @@ -1,7 +1,7 @@ use anyhow::Context; use semver::Version; use slog::{debug, Logger}; -use std::{collections::BTreeSet, sync::Arc}; +use std::{collections::BTreeSet, path::Path, sync::Arc}; use tokio::{ sync::{ mpsc::{UnboundedReceiver, UnboundedSender}, @@ -52,9 +52,9 @@ use mithril_persistence::{ use super::{DependenciesBuilderError, EpochServiceWrapper, Result}; use crate::{ artifact_builder::{ - CardanoDatabaseArtifactBuilder, CardanoImmutableFilesFullArtifactBuilder, - CardanoStakeDistributionArtifactBuilder, CardanoTransactionsArtifactBuilder, - MithrilStakeDistributionArtifactBuilder, + AncillaryArtifactBuilder, CardanoDatabaseArtifactBuilder, + CardanoImmutableFilesFullArtifactBuilder, CardanoStakeDistributionArtifactBuilder, + CardanoTransactionsArtifactBuilder, MithrilStakeDistributionArtifactBuilder, }, configuration::ExecutionEnvironment, database::repository::{ @@ -64,7 +64,7 @@ use crate::{ }, entities::AggregatorEpochSettings, event_store::{EventMessage, EventStore, TransmitterService}, - file_uploaders::GcpUploader, + file_uploaders::{FileUploader, GcpUploader}, http_server::routes::router::{self, RouterConfig, RouterState}, services::{ AggregatorSignableSeedBuilder, AggregatorUpkeepService, BufferedCertifierService, @@ -78,8 +78,8 @@ use crate::{ tools::{CExplorerSignerRetriever, GenesisToolsDependency, SignersImporter}, AggregatorConfig, AggregatorRunner, AggregatorRuntime, CompressedArchiveSnapshotter, Configuration, DependencyContainer, DumbSnapshotter, DumbUploader, EpochSettingsStorer, - FileUploader, LocalUploader, MetricsService, MithrilSignerRegisterer, MultiSigner, - MultiSignerImpl, SingleSignatureAuthenticator, SnapshotUploaderType, Snapshotter, + LocalUploader, MetricsService, MithrilSignerRegisterer, MultiSigner, MultiSignerImpl, + SingleSignatureAuthenticator, SnapshotUploaderType, Snapshotter, SnapshotterCompressionAlgorithm, VerificationKeyStorer, }; @@ -470,7 +470,7 @@ impl DependenciesBuilder { } SnapshotUploaderType::Local => Ok(Arc::new(LocalUploader::new( self.configuration.get_server_url(), - &self.configuration.snapshot_directory, + &self.configuration.get_snapshot_dir()?, logger, ))), } @@ -823,7 +823,7 @@ impl DependenciesBuilder { ExecutionEnvironment::Production => { let ongoing_snapshot_directory = self .configuration - .snapshot_directory + .get_snapshot_dir()? .join("pending_snapshot"); let algorithm = match self.configuration.snapshot_compression_algorithm { @@ -1182,6 +1182,39 @@ impl DependenciesBuilder { Ok(self.signable_seed_builder.as_ref().cloned().unwrap()) } + fn create_cardano_database_artifact_builder( + &self, + logger: &Logger, + cardano_node_version: Version, + ) -> Result { + let artifacts_dir = Path::new("cardano-database").join("ancillary"); + let snapshot_dir = self + .configuration + .get_snapshot_dir()? + .join(artifacts_dir.clone()); + std::fs::create_dir_all(snapshot_dir.clone()).map_err(|e| { + DependenciesBuilderError::Initialization { + message: format!("Cannot create '{artifacts_dir:?}' directory."), + error: Some(e.into()), + } + })?; + let local_uploader = LocalUploader::new( + self.configuration.get_server_url(), + &snapshot_dir, + logger.clone(), + ); + let ancillary_builder = Arc::new(AncillaryArtifactBuilder::new(vec![Arc::new( + local_uploader, + )])); + + Ok(CardanoDatabaseArtifactBuilder::new( + self.configuration.db_directory.clone(), + &cardano_node_version, + self.configuration.snapshot_compression_algorithm, + ancillary_builder, + )) + } + async fn build_signed_entity_service(&mut self) -> Result> { let logger = self.root_logger(); let signed_entity_storer = self.build_signed_entity_storer().await?; @@ -1209,11 +1242,8 @@ impl DependenciesBuilder { let stake_store = self.get_stake_store().await?; let cardano_stake_distribution_artifact_builder = Arc::new(CardanoStakeDistributionArtifactBuilder::new(stake_store)); - let cardano_database_artifact_builder = Arc::new(CardanoDatabaseArtifactBuilder::new( - self.configuration.db_directory.clone(), - &cardano_node_version, - self.configuration.snapshot_compression_algorithm, - )); + let cardano_database_artifact_builder = + Arc::new(self.create_cardano_database_artifact_builder(&logger, cardano_node_version)?); let dependencies = SignedEntityServiceArtifactsDependencies::new( mithril_stake_distribution_artifact_builder, cardano_immutable_files_full_artifact_builder, @@ -1524,7 +1554,7 @@ impl DependenciesBuilder { .configuration .cardano_transactions_signing_config .clone(), - snapshot_directory: self.configuration.snapshot_directory.clone(), + snapshot_directory: self.configuration.get_snapshot_dir()?, cardano_node_version: self.configuration.cardano_node_version.clone(), }, ); @@ -1720,7 +1750,9 @@ impl DependenciesBuilder { #[cfg(test)] mod tests { - use mithril_common::entities::SignedEntityTypeDiscriminants; + use mithril_common::{entities::SignedEntityTypeDiscriminants, test_utils::TempDir}; + + use crate::test_tools::TestLogger; use super::*; @@ -1761,4 +1793,35 @@ mod tests { expected_activation, is_activated ); } + + #[test] + fn create_cardano_database_artifact_builder_creates_cardano_database_and_ancillary_directories_in_snapshot_directory( + ) { + let snapshot_directory = TempDir::create( + "builder", + "create_cardano_database_and_ancillary_directories", + ); + let ancillary_dir = snapshot_directory + .join("cardano-database") + .join("ancillary"); + let dep_builder = { + let config = Configuration { + snapshot_directory, + ..Configuration::new_sample() + }; + + DependenciesBuilder::new_with_stdout_logger(config) + }; + + assert!(!ancillary_dir.exists()); + + dep_builder + .create_cardano_database_artifact_builder( + &TestLogger::stdout(), + Version::parse("1.0.0").unwrap(), + ) + .unwrap(); + + assert!(ancillary_dir.exists()); + } } diff --git a/mithril-aggregator/src/file_uploaders/local_uploader.rs b/mithril-aggregator/src/file_uploaders/local_uploader.rs index b6a2c7933f6..48499714815 100644 --- a/mithril-aggregator/src/file_uploaders/local_uploader.rs +++ b/mithril-aggregator/src/file_uploaders/local_uploader.rs @@ -124,4 +124,21 @@ mod tests { .join(archive.file_name().unwrap()) .exists()); } + + #[tokio::test] + async fn should_error_if_path_is_a_directory() { + let source_dir = tempdir().unwrap(); + let digest = "41e27b9ed5a32531b95b2b7ff3c0757591a06a337efaf19a524a998e348028e7"; + create_fake_archive(source_dir.path(), digest); + let target_dir = tempdir().unwrap(); + let uploader = LocalUploader::new( + "http://test.com:8080/".to_string(), + target_dir.path(), + TestLogger::stdout(), + ); + uploader + .upload(source_dir.path()) + .await + .expect_err("Uploading a directory should fail"); + } } diff --git a/mithril-common/Cargo.toml b/mithril-common/Cargo.toml index 724cd4ae289..460cfc49609 100644 --- a/mithril-common/Cargo.toml +++ b/mithril-common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-common" -version = "0.4.96" +version = "0.4.97" description = "Common types, interfaces, and utilities for Mithril nodes." authors = { workspace = true } edition = { workspace = true } diff --git a/mithril-common/src/entities/cardano_database.rs b/mithril-common/src/entities/cardano_database.rs index 9c6b5f0284e..cbaf541837b 100644 --- a/mithril-common/src/entities/cardano_database.rs +++ b/mithril-common/src/entities/cardano_database.rs @@ -1,5 +1,6 @@ use semver::Version; use serde::{Deserialize, Serialize}; +use strum::EnumDiscriminants; use crate::{ entities::{CardanoDbBeacon, CompressionAlgorithm}, @@ -51,34 +52,55 @@ impl CardanoDatabaseSnapshot { } } +/// Locations of the the immutable file digests. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case", tag = "type")] -enum DigestLocation { - Aggregator { uri: String }, - CloudStorage { uri: String }, +pub enum DigestLocation { + /// Aggregator digest route location. + Aggregator { + /// URI of the aggregator digests route location. + uri: String, + }, + /// Cloud storage location. + CloudStorage { + /// URI of the cloud storage location. + uri: String, + }, } +/// Locations of the ancillary files. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case", tag = "type")] -enum ImmutablesLocation { - CloudStorage { uri: String }, +pub enum ImmutablesLocation { + /// Cloud storage location. + CloudStorage { + /// URI of the cloud storage location. + uri: String, + }, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +/// Locations of the ancillary files. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, EnumDiscriminants)] #[serde(rename_all = "snake_case", tag = "type")] -enum AncillaryLocation { - CloudStorage { uri: String }, +pub enum AncillaryLocation { + /// Cloud storage location. + CloudStorage { + /// URI of the cloud storage location. + uri: String, + }, } /// Locations of the Cardano database related files. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct ArtifactsLocations { - /// Locations of the the immutable file digests. - digest: Vec, + /// Locations of the immutable file digests. + pub digest: Vec, + /// Locations of the immutable files. - immutables: Vec, + pub immutables: Vec, + /// Locations of the ancillary files. - ancillary: Vec, + pub ancillary: Vec, } #[typetag::serde] diff --git a/mithril-common/src/entities/mod.rs b/mithril-common/src/entities/mod.rs index ee48fb0e0dd..a0b36e9678c 100644 --- a/mithril-common/src/entities/mod.rs +++ b/mithril-common/src/entities/mod.rs @@ -33,7 +33,9 @@ mod type_alias; pub use block_number::BlockNumber; pub use block_range::{BlockRange, BlockRangeLength, BlockRangesSequence}; pub use cardano_chain_point::{BlockHash, ChainPoint}; -pub use cardano_database::{ArtifactsLocations, CardanoDatabaseSnapshot}; +pub use cardano_database::{ + AncillaryLocation, AncillaryLocationDiscriminants, ArtifactsLocations, CardanoDatabaseSnapshot, +}; pub use cardano_db_beacon::CardanoDbBeacon; pub use cardano_network::CardanoNetwork; pub use cardano_stake_distribution::CardanoStakeDistribution;