From c9746e1e72469fa8e68d884d35c9eb724c5e271f Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 1 Oct 2024 21:10:59 -0700 Subject: [PATCH] Standalone 9/N: Implement and test Sink and SinkCreator types (#301) --- src/streaming/CMakeLists.txt | 8 + src/streaming/acquire.zarr.cpp | 6 +- src/streaming/file.sink.cpp | 30 ++ src/streaming/file.sink.hh | 22 + src/streaming/s3.connection.cpp | 42 +- src/streaming/s3.connection.hh | 2 +- src/streaming/s3.sink.cpp | 205 ++++++++ src/streaming/s3.sink.hh | 74 +++ src/streaming/sink.cpp | 12 + src/streaming/sink.creator.cpp | 488 ++++++++++++++++++ src/streaming/sink.creator.hh | 181 +++++++ src/streaming/sink.hh | 31 ++ src/streaming/zarr.dimension.hh | 1 + tests/unit-tests/CMakeLists.txt | 4 + ...array-dimensions-chunk-internal-offset.cpp | 152 +++--- .../array-dimensions-chunk-lattice-index.cpp | 102 ++-- ...array-dimensions-shard-index-for-chunk.cpp | 290 +++++------ .../array-dimensions-shard-internal-index.cpp | 48 +- .../array-dimensions-tile-group-offset.cpp | 154 +++--- tests/unit-tests/create-stream.cpp | 2 +- tests/unit-tests/file-sink-write.cpp | 53 ++ .../s3-connection-bucket-exists.cpp | 2 +- ...on-object-exists-check-false-positives.cpp | 2 +- tests/unit-tests/s3-connection-put-object.cpp | 2 +- .../s3-connection-upload-multipart-object.cpp | 2 +- tests/unit-tests/s3-sink-write.cpp | 127 +++++ .../sink-creator-make-data-sinks.cpp | 280 ++++++++++ .../sink-creator-make-metadata-sinks.cpp | 216 ++++++++ .../thread-pool-push-to-job-queue.cpp | 54 +- tests/unit-tests/unit.test.macros.hh | 25 +- 30 files changed, 2175 insertions(+), 442 deletions(-) create mode 100644 src/streaming/file.sink.cpp create mode 100644 src/streaming/file.sink.hh create mode 100644 src/streaming/s3.sink.cpp create mode 100644 src/streaming/s3.sink.hh create mode 100644 src/streaming/sink.cpp create mode 100644 src/streaming/sink.creator.cpp create mode 100644 src/streaming/sink.creator.hh create mode 100644 src/streaming/sink.hh create mode 100644 tests/unit-tests/file-sink-write.cpp create mode 100644 tests/unit-tests/s3-sink-write.cpp create mode 100644 tests/unit-tests/sink-creator-make-data-sinks.cpp create mode 100644 tests/unit-tests/sink-creator-make-metadata-sinks.cpp diff --git a/src/streaming/CMakeLists.txt b/src/streaming/CMakeLists.txt index 598fffe5..85f4a97e 100644 --- a/src/streaming/CMakeLists.txt +++ b/src/streaming/CMakeLists.txt @@ -15,6 +15,14 @@ add_library(${tgt} thread.pool.cpp s3.connection.hh s3.connection.cpp + sink.hh + sink.cpp + file.sink.hh + file.sink.cpp + s3.sink.hh + s3.sink.cpp + sink.creator.hh + sink.creator.cpp ) target_include_directories(${tgt} diff --git a/src/streaming/acquire.zarr.cpp b/src/streaming/acquire.zarr.cpp index 9b1f46e2..bbc57dab 100644 --- a/src/streaming/acquire.zarr.cpp +++ b/src/streaming/acquire.zarr.cpp @@ -34,7 +34,7 @@ extern "C" try { Logger::set_log_level(level); } catch (const std::exception& e) { - LOG_ERROR("Error setting log level: %s", e.what()); + LOG_ERROR("Error setting log level: ", e.what()); return ZarrStatusCode_InternalError; } return ZarrStatusCode_Success; @@ -94,7 +94,7 @@ extern "C" { EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); EXPECT_VALID_ARGUMENT(dimension_count >= 3, - "Invalid dimension count: %zu", + "Invalid dimension count: ", dimension_count); ZarrDimensionProperties* dimensions = nullptr; @@ -160,7 +160,7 @@ extern "C" try { *bytes_out = stream->append(data, bytes_in); } catch (const std::exception& e) { - LOG_ERROR("Error appending data: %s", e.what()); + LOG_ERROR("Error appending data: ", e.what()); return ZarrStatusCode_InternalError; } diff --git a/src/streaming/file.sink.cpp b/src/streaming/file.sink.cpp new file mode 100644 index 00000000..4db4f3a6 --- /dev/null +++ b/src/streaming/file.sink.cpp @@ -0,0 +1,30 @@ +#include "file.sink.hh" + +#include + +namespace fs = std::filesystem; + +zarr::FileSink::FileSink(std::string_view filename) +: file_(filename.data(), std::ios::binary | std::ios::trunc) +{ +} + +bool +zarr::FileSink::write(size_t offset, std::span data) +{ + const auto bytes_of_buf = data.size(); + if (data.data() == nullptr || bytes_of_buf == 0) { + return true; + } + + file_.seekp(offset); + file_.write(reinterpret_cast(data.data()), bytes_of_buf); + return true; +} + +bool +zarr::FileSink::flush_() +{ + file_.flush(); + return true; +} diff --git a/src/streaming/file.sink.hh b/src/streaming/file.sink.hh new file mode 100644 index 00000000..0bd2181f --- /dev/null +++ b/src/streaming/file.sink.hh @@ -0,0 +1,22 @@ +#pragma once + +#include "sink.hh" + +#include +#include + +namespace zarr { +class FileSink : public Sink +{ + public: + explicit FileSink(std::string_view filename); + + bool write(size_t offset, std::span data) override; + + protected: + bool flush_() override; + + private: + std::ofstream file_; +}; +} // namespace zarr diff --git a/src/streaming/s3.connection.cpp b/src/streaming/s3.connection.cpp index c9ffbf88..d4ea3f37 100644 --- a/src/streaming/s3.connection.cpp +++ b/src/streaming/s3.connection.cpp @@ -63,18 +63,19 @@ zarr::S3Connection::put_object(std::string_view bucket_name, data.size()); std::basic_istream stream(&buffer); - LOG_DEBUG( - "Putting object %s in bucket %s", object_name.data(), bucket_name.data()); + LOG_DEBUG("Putting object ", object_name, " in bucket ", bucket_name); minio::s3::PutObjectArgs args(stream, static_cast(data.size()), 0); args.bucket = bucket_name; args.object = object_name; auto response = client_->PutObject(args); if (!response) { - LOG_ERROR("Failed to put object %s in bucket %s: %s", - object_name.data(), - bucket_name.data(), - response.Error().String().c_str()); + LOG_ERROR("Failed to put object ", + object_name, + " in bucket ", + bucket_name, + ": ", + response.Error().String()); return {}; } @@ -88,19 +89,19 @@ zarr::S3Connection::delete_object(std::string_view bucket_name, EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); EXPECT(!object_name.empty(), "Object name must not be empty."); - LOG_DEBUG("Deleting object %s from bucket %s", - object_name.data(), - bucket_name.data()); + LOG_DEBUG("Deleting object ", object_name, " from bucket ", bucket_name); minio::s3::RemoveObjectArgs args; args.bucket = bucket_name; args.object = object_name; auto response = client_->RemoveObject(args); if (!response) { - LOG_ERROR("Failed to delete object %s from bucket %s: %s", - object_name.data(), - bucket_name.data(), - response.Error().String().c_str()); + LOG_ERROR("Failed to delete object ", + object_name, + " from bucket ", + bucket_name, + ": ", + response.Error().String()); return false; } @@ -191,9 +192,8 @@ zarr::S3Connection::complete_multipart_object( EXPECT(!upload_id.empty(), "Upload id must not be empty."); EXPECT(!parts.empty(), "Parts list must not be empty."); - LOG_DEBUG("Completing multipart object %s in bucket %s", - object_name.data(), - bucket_name.data()); + LOG_DEBUG( + "Completing multipart object ", object_name, " in bucket ", bucket_name); minio::s3::CompleteMultipartUploadArgs args; args.bucket = bucket_name; args.object = object_name; @@ -202,10 +202,12 @@ zarr::S3Connection::complete_multipart_object( auto response = client_->CompleteMultipartUpload(args); if (!response) { - LOG_ERROR("Failed to complete multipart object %s in bucket %s: %s", - object_name.data(), - bucket_name.data(), - response.Error().String().c_str()); + LOG_ERROR("Failed to complete multipart object ", + object_name, + " in bucket ", + bucket_name, + ": ", + response.Error().String()); return false; } diff --git a/src/streaming/s3.connection.hh b/src/streaming/s3.connection.hh index 91e71f27..744fb933 100644 --- a/src/streaming/s3.connection.hh +++ b/src/streaming/s3.connection.hh @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/streaming/s3.sink.cpp b/src/streaming/s3.sink.cpp new file mode 100644 index 00000000..3ce664ba --- /dev/null +++ b/src/streaming/s3.sink.cpp @@ -0,0 +1,205 @@ +#include "macros.hh" +#include "s3.sink.hh" + +#include + +#ifdef min +#undef min +#endif + +zarr::S3Sink::S3Sink(std::string_view bucket_name, + std::string_view object_key, + std::shared_ptr connection_pool) + : bucket_name_{ bucket_name } + , object_key_{ object_key } + , connection_pool_{ connection_pool } +{ + EXPECT(!bucket_name_.empty(), "Bucket name must not be empty"); + EXPECT(!object_key_.empty(), "Object key must not be empty"); + EXPECT(connection_pool_, "Null pointer: connection_pool"); +} + +bool +zarr::S3Sink::flush_() +{ + if (is_multipart_upload_()) { + const auto& parts = multipart_upload_->parts; + if (nbytes_buffered_ > 0 && !flush_part_()) { + LOG_ERROR("Failed to upload part ", + parts.size() + 1, + " of object ", + object_key_); + return false; + } + if (!finalize_multipart_upload_()) { + LOG_ERROR("Failed to finalize multipart upload of object ", + object_key_); + return false; + } + } else if (nbytes_buffered_ > 0) { + if (!put_object_()) { + LOG_ERROR("Failed to upload object: ", object_key_); + return false; + } + } + + // cleanup + nbytes_buffered_ = 0; + + return true; +} + +bool +zarr::S3Sink::write(size_t offset, std::span data) +{ + if (data.data() == nullptr || data.empty()) { + return true; + } + + if (offset < nbytes_flushed_) { + LOG_ERROR("Cannot write data at offset ", + offset, + ", already flushed to ", + nbytes_flushed_); + return false; + } + nbytes_buffered_ = offset - nbytes_flushed_; + + size_t bytes_of_data = data.size(); + std::byte* data_ptr = data.data(); + while (bytes_of_data > 0) { + const auto bytes_to_write = + std::min(bytes_of_data, part_buffer_.size() - nbytes_buffered_); + + if (bytes_to_write) { + std::copy_n(data_ptr, + bytes_to_write, + part_buffer_.begin() + nbytes_buffered_); + nbytes_buffered_ += bytes_to_write; + data_ptr += bytes_to_write; + bytes_of_data -= bytes_to_write; + } + + if (nbytes_buffered_ == part_buffer_.size() && !flush_part_()) { + return false; + } + } + + return true; +} + +bool +zarr::S3Sink::put_object_() +{ + if (nbytes_buffered_ == 0) { + return false; + } + + auto connection = connection_pool_->get_connection(); + std::span data(reinterpret_cast(part_buffer_.data()), + nbytes_buffered_); + + bool retval = false; + try { + std::string etag = + connection->put_object(bucket_name_, object_key_, data); + EXPECT(!etag.empty(), "Failed to upload object: ", object_key_); + + retval = true; + + nbytes_flushed_ = nbytes_buffered_; + nbytes_buffered_ = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Error: ", exc.what()); + } + + // cleanup + connection_pool_->return_connection(std::move(connection)); + + return retval; +} + +bool +zarr::S3Sink::is_multipart_upload_() const +{ + return multipart_upload_.has_value(); +} + +void +zarr::S3Sink::create_multipart_upload_() +{ + if (!is_multipart_upload_()) { + multipart_upload_ = {}; + } + + if (!multipart_upload_->upload_id.empty()) { + return; + } + + multipart_upload_->upload_id = + connection_pool_->get_connection()->create_multipart_object(bucket_name_, + object_key_); +} + +bool +zarr::S3Sink::flush_part_() +{ + if (nbytes_buffered_ == 0) { + return false; + } + + auto connection = connection_pool_->get_connection(); + + create_multipart_upload_(); + + bool retval = false; + try { + auto& parts = multipart_upload_->parts; + + minio::s3::Part part; + part.number = static_cast(parts.size()) + 1; + + std::span data(reinterpret_cast(part_buffer_.data()), + nbytes_buffered_); + part.etag = + connection->upload_multipart_object_part(bucket_name_, + object_key_, + multipart_upload_->upload_id, + data, + part.number); + EXPECT(!part.etag.empty(), + "Failed to upload part ", + part.number, + " of object ", + object_key_); + + parts.push_back(part); + + retval = true; + } catch (const std::exception& exc) { + LOG_ERROR("Error: ", exc.what()); + } + + // cleanup + connection_pool_->return_connection(std::move(connection)); + nbytes_flushed_ += nbytes_buffered_; + nbytes_buffered_ = 0; + + return retval; +} + +bool +zarr::S3Sink::finalize_multipart_upload_() +{ + auto connection = connection_pool_->get_connection(); + + const auto& upload_id = multipart_upload_->upload_id; + const auto& parts = multipart_upload_->parts; + + bool retval = connection->complete_multipart_object( + bucket_name_, object_key_, upload_id, parts); + + connection_pool_->return_connection(std::move(connection)); + + return retval; +} diff --git a/src/streaming/s3.sink.hh b/src/streaming/s3.sink.hh new file mode 100644 index 00000000..1e2c9d9f --- /dev/null +++ b/src/streaming/s3.sink.hh @@ -0,0 +1,74 @@ +#pragma once + +#include "sink.hh" +#include "s3.connection.hh" + +#include + +#include +#include +#include + +namespace zarr { +class S3Sink : public Sink +{ + public: + S3Sink(std::string_view bucket_name, + std::string_view object_key, + std::shared_ptr connection_pool); + + bool write(size_t offset, std::span data) override; + + protected: + bool flush_() override; + + private: + struct MultiPartUpload + { + std::string upload_id; + std::list parts; + }; + + static constexpr size_t max_part_size_ = 5 << 20; + std::string bucket_name_; + std::string object_key_; + + std::shared_ptr connection_pool_; + + std::array part_buffer_; + size_t nbytes_buffered_{ 0 }; + size_t nbytes_flushed_{ 0 }; + + std::optional multipart_upload_; + + /** + * @brief Upload the object to S3. + * @return True if the object was successfully uploaded, otherwise false. + */ + [[nodiscard]] bool put_object_(); + + /** + * @brief Check if a multipart upload is in progress. + * @return True if a multipart upload is in progress, otherwise false. + */ + bool is_multipart_upload_() const; + + /** + * @brief Create a new multipart upload. + */ + void create_multipart_upload_(); + + /** + * @brief Flush the current part to S3. + * @return True if the part was successfully flushed, otherwise false. + */ + [[nodiscard]] bool flush_part_(); + + /** + * @brief Finalize the multipart upload. + * @returns True if a multipart upload was successfully finalized, + * otherwise false. + */ + [[nodiscard]] bool finalize_multipart_upload_(); +}; +} // namespace zarr diff --git a/src/streaming/sink.cpp b/src/streaming/sink.cpp new file mode 100644 index 00000000..9f755666 --- /dev/null +++ b/src/streaming/sink.cpp @@ -0,0 +1,12 @@ +#include "sink.hh" + +bool +zarr::finalize_sink(std::unique_ptr&& sink) +{ + if (!sink->flush_()) { + return false; + } + + sink.reset(); + return true; +} \ No newline at end of file diff --git a/src/streaming/sink.creator.cpp b/src/streaming/sink.creator.cpp new file mode 100644 index 00000000..2eadc35c --- /dev/null +++ b/src/streaming/sink.creator.cpp @@ -0,0 +1,488 @@ +#include "macros.hh" +#include "sink.creator.hh" +#include "file.sink.hh" +#include "s3.sink.hh" +#include "acquire.zarr.h" + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +zarr::SinkCreator::SinkCreator( + std::shared_ptr thread_pool_, + std::shared_ptr connection_pool) + : thread_pool_{ thread_pool_ } + , connection_pool_{ connection_pool } +{ +} + +std::unique_ptr +zarr::SinkCreator::make_sink(std::string_view file_path) +{ + if (file_path.starts_with("file://")) { + file_path = file_path.substr(7); + } + + EXPECT(!file_path.empty(), "File path must not be empty."); + + fs::path path(file_path); + EXPECT(!path.empty(), "Invalid file path: ", file_path); + + fs::path parent_path = path.parent_path(); + + if (!fs::is_directory(parent_path)) { + std::error_code ec; + if (!fs::create_directories(parent_path, ec)) { + LOG_ERROR("Failed to create directory '", parent_path, "': ", + ec.message()); + return nullptr; + } + } + + return std::make_unique(file_path); +} + +std::unique_ptr +zarr::SinkCreator::make_sink(std::string_view bucket_name, + std::string_view object_key) +{ + EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); + EXPECT(!object_key.empty(), "Object key must not be empty."); + EXPECT(connection_pool_, "S3 connection pool not provided."); + if (!bucket_exists_(bucket_name)) { + LOG_ERROR("Bucket '", bucket_name, "' does not exist."); + return nullptr; + } + + return std::make_unique(bucket_name, object_key, connection_pool_); +} + +bool +zarr::SinkCreator::make_data_sinks( + std::string_view base_path, + std::shared_ptr dimensions, + const std::function& parts_along_dimension, + std::vector>& part_sinks) +{ + if (base_path.starts_with("file://")) { + base_path = base_path.substr(7); + } + + EXPECT(!base_path.empty(), "Base path must not be empty."); + + std::queue paths; + try { + paths = make_data_sink_paths_( + base_path, dimensions, parts_along_dimension, true); + } catch (const std::exception& exc) { + LOG_ERROR("Failed to create dataset paths: ", exc.what()); + return false; + } + + return make_files_(paths, part_sinks); +} + +bool +zarr::SinkCreator::make_data_sinks( + std::string_view bucket_name, + std::string_view base_path, + std::shared_ptr dimensions, + const std::function& parts_along_dimension, + std::vector>& part_sinks) +{ + EXPECT(!base_path.empty(), "Base path must not be empty."); + + auto paths = make_data_sink_paths_( + base_path, dimensions, parts_along_dimension, false); + + return make_s3_objects_(bucket_name, paths, part_sinks); +} + +bool +zarr::SinkCreator::make_metadata_sinks( + size_t version, + std::string_view base_path, + std::unordered_map>& metadata_sinks) +{ + if (base_path.starts_with("file://")) { + base_path = base_path.substr(7); + } + EXPECT(!base_path.empty(), "Base path must not be empty."); + + std::vector file_paths = + make_metadata_sink_paths_(version, base_path, true); + + return make_files_(base_path.data(), file_paths, metadata_sinks); +} + +bool +zarr::SinkCreator::make_metadata_sinks( + size_t version, + std::string_view bucket_name, + std::string_view base_path, + std::unordered_map>& metadata_sinks) +{ + EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); + EXPECT(!base_path.empty(), "Base path must not be empty."); + if (!bucket_exists_(bucket_name)) { + LOG_ERROR("Bucket '", bucket_name, "' does not exist."); + return false; + } + + auto file_paths = make_metadata_sink_paths_(version, base_path, false); + return make_s3_objects_(bucket_name, base_path, file_paths, metadata_sinks); +} + +std::queue +zarr::SinkCreator::make_data_sink_paths_( + std::string_view base_path, + std::shared_ptr dimensions, + const std::function& parts_along_dimension, + bool create_directories) +{ + std::queue paths; + paths.emplace(base_path); + + if (create_directories) { + EXPECT( + make_dirs_(paths), "Failed to create directory '", base_path, "'."); + } + + // create intermediate paths + for (auto i = 1; // skip the last dimension + i < dimensions->ndims() - 1; // skip the x dimension + ++i) { + const auto& dim = dimensions->at(i); + const auto n_parts = parts_along_dimension(dim); + CHECK(n_parts); + + auto n_paths = paths.size(); + for (auto j = 0; j < n_paths; ++j) { + const auto path = paths.front(); + paths.pop(); + + for (auto k = 0; k < n_parts; ++k) { + const auto kstr = std::to_string(k); + paths.push(path + (path.empty() ? kstr : "/" + kstr)); + } + } + + if (create_directories) { + EXPECT(make_dirs_(paths), + "Failed to create directories for dimension '", + dim.name, + "'."); + } + } + + // create final paths + { + const auto& dim = dimensions->width_dim(); + const auto n_parts = parts_along_dimension(dim); + CHECK(n_parts); + + auto n_paths = paths.size(); + for (auto i = 0; i < n_paths; ++i) { + const auto path = paths.front(); + paths.pop(); + for (auto j = 0; j < n_parts; ++j) + paths.push(path + "/" + std::to_string(j)); + } + } + + return paths; +} + +std::vector +zarr::SinkCreator::make_metadata_sink_paths_(size_t version, + std::string_view base_path, + bool create_directories) +{ + std::vector paths; + + switch (version) { + case ZarrVersion_2: + paths.emplace_back(".zattrs"); + paths.emplace_back(".zgroup"); + paths.emplace_back("0/.zattrs"); + paths.emplace_back("acquire.json"); + break; + case ZarrVersion_3: + paths.emplace_back("zarr.json"); + paths.emplace_back("meta/root.group.json"); + paths.emplace_back("meta/acquire.json"); + break; + default: + throw std::runtime_error("Invalid Zarr version " + + std::to_string(static_cast(version))); + } + + if (create_directories) { + std::queue dir_paths; + dir_paths.emplace(base_path); + EXPECT(make_dirs_(dir_paths), + "Failed to create metadata directories."); + dir_paths.pop(); // remove the base path + + std::unordered_set parent_paths; + for (const auto& path : paths) { + fs::path parent = fs::path(path).parent_path(); + if (!parent.empty()) { + parent_paths.emplace((fs::path(base_path) / parent).string()); + } + } + + for (const auto& dir_path : parent_paths) { + dir_paths.push(dir_path); + } + + if (!dir_paths.empty()) { + EXPECT(make_dirs_(dir_paths), + "Failed to create metadata directories."); + } + } + + return paths; +} + +bool +zarr::SinkCreator::make_dirs_(std::queue& dir_paths) +{ + if (dir_paths.empty()) { + return true; + } + + std::atomic all_successful = 1; + + const auto n_dirs = dir_paths.size(); + std::latch latch(n_dirs); + + for (auto i = 0; i < n_dirs; ++i) { + const auto dirname = dir_paths.front(); + dir_paths.pop(); + + EXPECT(thread_pool_->push_job( + [dirname, &latch, &all_successful](std::string& err) -> bool { + if (dirname.empty()) { + err = "Directory name must not be empty."; + latch.count_down(); + all_successful.fetch_and(0); + return false; + } + + if (fs::is_directory(dirname)) { + latch.count_down(); + return true; + } else if (fs::exists(dirname)) { + err = + "'" + dirname + "' exists but is not a directory"; + latch.count_down(); + all_successful.fetch_and(0); + return false; + } + + if (all_successful) { + std::error_code ec; + if (!fs::create_directories(dirname, ec)) { + err = "Failed to create directory '" + dirname + + "': " + ec.message(); + latch.count_down(); + all_successful.fetch_and(0); + return false; + } + } + + latch.count_down(); + return true; + }), + "Failed to push job to thread pool."); + + dir_paths.push(dirname); + } + + latch.wait(); + + return (bool)all_successful; +} + +bool +zarr::SinkCreator::make_files_(std::queue& file_paths, + std::vector>& sinks) +{ + if (file_paths.empty()) { + return true; + } + + std::atomic all_successful = 1; + + const auto n_files = file_paths.size(); + sinks.resize(n_files); + std::fill(sinks.begin(), sinks.end(), nullptr); + std::latch latch(n_files); + + for (auto i = 0; i < n_files; ++i) { + const auto filename = file_paths.front(); + file_paths.pop(); + + std::unique_ptr* psink = sinks.data() + i; + + EXPECT(thread_pool_->push_job( + [filename, psink, &latch, &all_successful]( + std::string& err) -> bool { + bool success = false; + + try { + if (all_successful) { + *psink = std::make_unique(filename); + } + success = true; + } catch (const std::exception& exc) { + err = "Failed to create file '" + filename + + "': " + exc.what(); + } catch (...) { + err = "Failed to create file '" + filename + + "': (unknown)."; + } + + latch.count_down(); + all_successful.fetch_and((char)success); + + return success; + }), + "Failed to push job to thread pool."); + } + + latch.wait(); + + return (bool)all_successful; +} + +bool +zarr::SinkCreator::make_files_( + const std::string& base_dir, + const std::vector& file_paths, + std::unordered_map>& sinks) +{ + if (file_paths.empty()) { + return true; + } + + std::atomic all_successful = 1; + + const auto n_files = file_paths.size(); + std::latch latch(n_files); + + sinks.clear(); + for (const auto& filename : file_paths) { + sinks[filename] = nullptr; + std::unique_ptr* psink = &sinks[filename]; + + const std::string prefix = base_dir.empty() ? "" : base_dir + "/"; + const auto file_path = prefix + filename; + + EXPECT(thread_pool_->push_job( + [filename = file_path, psink, &latch, &all_successful]( + std::string& err) -> bool { + bool success = false; + + try { + if (all_successful) { + *psink = std::make_unique(filename); + } + success = true; + } catch (const std::exception& exc) { + err = "Failed to create file '" + filename + + "': " + exc.what(); + } catch (...) { + err = "Failed to create file '" + filename + + "': (unknown)."; + } + + latch.count_down(); + all_successful.fetch_and((char)success); + + return success; + }), + "Failed to push job to thread pool."); + } + + latch.wait(); + + return (bool)all_successful; +} + +bool +zarr::SinkCreator::bucket_exists_(std::string_view bucket_name) +{ + CHECK(!bucket_name.empty()); + EXPECT(connection_pool_, "S3 connection pool not provided."); + + auto conn = connection_pool_->get_connection(); + bool bucket_exists = conn->bucket_exists(bucket_name); + + connection_pool_->return_connection(std::move(conn)); + + return bucket_exists; +} + +bool +zarr::SinkCreator::make_s3_objects_(std::string_view bucket_name, + std::queue& object_keys, + std::vector>& sinks) +{ + if (object_keys.empty()) { + return true; + } + + if (bucket_name.empty()) { + LOG_ERROR("Bucket name not provided."); + return false; + } + if (!connection_pool_) { + LOG_ERROR("S3 connection pool not provided."); + return false; + } + + const auto n_objects = object_keys.size(); + sinks.resize(n_objects); + for (auto i = 0; i < n_objects; ++i) { + sinks[i] = std::make_unique( + bucket_name, object_keys.front(), connection_pool_); + object_keys.pop(); + } + + return true; +} + +bool +zarr::SinkCreator::make_s3_objects_( + std::string_view bucket_name, + std::string_view base_path, + std::vector& object_keys, + std::unordered_map>& sinks) +{ + if (object_keys.empty()) { + return true; + } + + if (bucket_name.empty()) { + LOG_ERROR("Bucket name not provided."); + return false; + } + + if (!connection_pool_) { + LOG_ERROR("S3 connection pool not provided."); + return false; + } + + sinks.clear(); + for (const auto& key : object_keys) { + sinks[key] = std::make_unique( + bucket_name, std::string(base_path) + "/" + key, connection_pool_); + } + + return true; +} diff --git a/src/streaming/sink.creator.hh b/src/streaming/sink.creator.hh new file mode 100644 index 00000000..85c29e4d --- /dev/null +++ b/src/streaming/sink.creator.hh @@ -0,0 +1,181 @@ +#pragma once + +#include "zarr.dimension.hh" +#include "sink.hh" +#include "thread.pool.hh" +#include "s3.connection.hh" + +#include +#include +#include + +namespace zarr { +class SinkCreator +{ + public: + SinkCreator(std::shared_ptr thread_pool_, + std::shared_ptr connection_pool); + ~SinkCreator() noexcept = default; + + /** + * @brief Create a sink from a file path. + * @param file_path The path to the file. + * @return Pointer to the sink created, or nullptr if the file cannot be + * opened. + * @throws std::runtime_error if the file path is not valid. + */ + static std::unique_ptr make_sink(std::string_view file_path); + + /** + * @brief Create a sink from an S3 bucket name and object key. + * @param bucket_name The name of the bucket in which the object is stored. + * @param object_key The key of the object to write to. + * @return Pointer to the sink created, or nullptr if the bucket does not + * exist. + * @throws std::runtime_error if the bucket name or object key is not valid, + * or if there is no connection pool. + */ + std::unique_ptr make_sink(std::string_view bucket_name, + std::string_view object_key); + + /** + * @brief Create a collection of file sinks for a Zarr dataset. + * @param[in] base_path The path to the base directory for the dataset. + * @param[in] dimensions The dimensions of the dataset. + * @param[in] parts_along_dimension Function to determine the number of + * parts (i.e., shards or chunks) along a dimension. + * @param[out] part_sinks The sinks created. + * @return True iff all file sinks were created successfully. + * @throws std::runtime_error if @p base_path is not valid, or if the number + * of parts along a dimension is zero. + */ + [[nodiscard]] bool make_data_sinks( + std::string_view base_path, + std::shared_ptr dimensions, + const std::function& parts_along_dimension, + std::vector>& part_sinks); + + /** + * @brief Create a collection of S3 sinks for a Zarr dataset. + * @param[in] bucket_name The name of the bucket in which the dataset is + * stored. + * @param[in] base_path The path to the base directory for the dataset. + * @param[in] dimensions The dimensions of the dataset. + * @param[in] parts_along_dimension Function to determine the number of + * parts (i.e., shards or chunks) along a dimension. + * @param[out] part_sinks The sinks created. + * @return True iff all file sinks were created successfully. + */ + [[nodiscard]] bool make_data_sinks( + std::string_view bucket_name, + std::string_view base_path, + std::shared_ptr dimensions, + const std::function& parts_along_dimension, + std::vector>& part_sinks); + + /** + * @brief Create a collection of metadata sinks for a Zarr dataset. + * @param[in] version The Zarr version. + * @param[in] base_path The base URI for the dataset. + * @param[out] metadata_sinks The sinks created, keyed by path. + * @return True iff all metadata sinks were created successfully. + * @throws std::runtime_error if @p base_uri is not valid, or if, for S3 + * sinks, the bucket does not exist. + */ + [[nodiscard]] bool make_metadata_sinks( + size_t version, + std::string_view base_path, + std::unordered_map>& metadata_sinks); + + /** + * @brief + * @param version + * @param bucket_name + * @param base_path + * @param metadata_sinks + * @return + * @throws std::runtime_error if @p version is invalid, if @p bucket_name is + * empty or does not exist, or if @p base_path is empty. + */ + [[nodiscard]] bool make_metadata_sinks( + size_t version, + std::string_view bucket_name, + std::string_view base_path, + std::unordered_map>& metadata_sinks); + + private: + std::shared_ptr thread_pool_; + std::shared_ptr connection_pool_; // could be null + + /** + * @brief Construct the paths for a Zarr dataset. + * @param base_path The base path for the dataset. + * @param dimensions The dimensions of the dataset. + * @param parts_along_dimension Function to determine the number of parts + * @param create_directories Whether to create intermediate directories. + * @return A queue of paths to the dataset components. + * @throws std::runtime_error if intermediate directories cannot be created, + * or if the number of parts along a dimension is zero. + */ + std::queue make_data_sink_paths_( + std::string_view base_path, + std::shared_ptr dimensions, + const std::function& parts_along_dimension, + bool create_directories); + + std::vector make_metadata_sink_paths_( + size_t version, + std::string_view base_path, + bool create_directories); + + /// @brief Parallel create a collection of directories. + /// @param[in] dir_paths The directories to create. + /// @return True iff all directories were created successfully. + [[nodiscard]] bool make_dirs_(std::queue& dir_paths); + + /// @brief Parallel create a collection of files. + /// @param[in,out] file_paths The files to create. Unlike `make_dirs_`, + /// this function drains the queue. + /// @param[out] files The files created. + /// @return True iff all files were created successfully. + [[nodiscard]] bool make_files_(std::queue& file_paths, + std::vector>& sinks); + + /// @brief Parallel create a collection of files, keyed by path. + /// @param[in] base_dir The base directory for the files. + /// @param[in] file_paths Paths to the files to create, relative to @p + /// base_dir. + /// @param[out] sinks The sinks created, keyed by path. + /// @return True iff all files were created successfully. + [[nodiscard]] bool make_files_( + const std::string& base_dir, + const std::vector& file_paths, + std::unordered_map>& sinks); + + /// @brief Check whether an S3 bucket exists. + /// @param[in] bucket_name The name of the bucket to check. + /// @return True iff the bucket exists. + bool bucket_exists_(std::string_view bucket_name); + + /// @brief Create a collection of S3 objects. + /// @param[in] bucket_name The name of the bucket. + /// @param[in,out] object_keys The keys of the objects to create. + /// @param[out] sinks The sinks created. + /// @return True iff all S3 objects were created successfully. + [[nodiscard]] bool make_s3_objects_( + std::string_view bucket_name, + std::queue& object_keys, + std::vector>& sinks); + + /// @brief Create a collection of S3 objects, keyed by object key. + /// @param[in] bucket_name The name of the bucket. + /// @param[in] object_keys The keys of the objects to create. + /// @param[out] sinks The sinks created, keyed by object key. + /// @return True iff all S3 objects were created successfully. + [[nodiscard]] bool make_s3_objects_( + std::string_view bucket_name, + std::string_view base_path, + std::vector& object_keys, + std::unordered_map>& sinks); +}; +} // namespace zarr diff --git a/src/streaming/sink.hh b/src/streaming/sink.hh new file mode 100644 index 00000000..ba578c78 --- /dev/null +++ b/src/streaming/sink.hh @@ -0,0 +1,31 @@ +#pragma once + +#include // size_t, std::byte +#include // std::unique_ptr +#include // std::span + +namespace zarr { +class Sink +{ + public: + virtual ~Sink() = default; + + /** + * @brief Write data to the sink. + * @param offset The offset in the sink to write to. + * @param buf The buffer to write to the sink. + * @param bytes_of_buf The number of bytes to write from @p buf. + * @return True if the write was successful, false otherwise. + */ + [[nodiscard]] virtual bool write(size_t offset, + std::span buf) = 0; + + protected: + [[nodiscard]] virtual bool flush_() = 0; + + friend bool finalize_sink(std::unique_ptr&& sink); +}; + +bool +finalize_sink(std::unique_ptr&& sink); +} // namespace zarr diff --git a/src/streaming/zarr.dimension.hh b/src/streaming/zarr.dimension.hh index ff7a58c1..a28d7530 100644 --- a/src/streaming/zarr.dimension.hh +++ b/src/streaming/zarr.dimension.hh @@ -36,6 +36,7 @@ class ArrayDimensions size_t ndims() const; const ZarrDimension& operator[](size_t idx) const; + const ZarrDimension& at(size_t idx) const { return operator[](idx); } const ZarrDimension& final_dim() const; const ZarrDimension& height_dim() const; diff --git a/tests/unit-tests/CMakeLists.txt b/tests/unit-tests/CMakeLists.txt index f5381a00..7740470b 100644 --- a/tests/unit-tests/CMakeLists.txt +++ b/tests/unit-tests/CMakeLists.txt @@ -12,6 +12,10 @@ set(tests s3-connection-object-exists-check-false-positives s3-connection-put-object s3-connection-upload-multipart-object + file-sink-write + s3-sink-write + sink-creator-make-metadata-sinks + sink-creator-make-data-sinks ) foreach (name ${tests}) diff --git a/tests/unit-tests/array-dimensions-chunk-internal-offset.cpp b/tests/unit-tests/array-dimensions-chunk-internal-offset.cpp index f22f569b..53702fd4 100644 --- a/tests/unit-tests/array-dimensions-chunk-internal-offset.cpp +++ b/tests/unit-tests/array-dimensions-chunk-internal-offset.cpp @@ -18,82 +18,82 @@ main() ArrayDimensions dimensions(std::move(dims), ZarrDataType_uint16); try { - EXPECT_INT_EQ(dimensions.chunk_internal_offset(0), 0); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(1), 512); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(2), 0); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(3), 512); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(4), 0); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(5), 1024); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(6), 1536); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(7), 1024); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(8), 1536); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(9), 1024); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(10), 0); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(11), 512); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(12), 0); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(13), 512); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(14), 0); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(15), 2048); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(16), 2560); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(17), 2048); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(18), 2560); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(19), 2048); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(20), 3072); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(21), 3584); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(22), 3072); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(23), 3584); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(24), 3072); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(25), 2048); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(26), 2560); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(27), 2048); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(28), 2560); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(29), 2048); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(30), 4096); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(31), 4608); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(32), 4096); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(33), 4608); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(34), 4096); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(35), 5120); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(36), 5632); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(37), 5120); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(38), 5632); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(39), 5120); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(40), 4096); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(41), 4608); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(42), 4096); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(43), 4608); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(44), 4096); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(45), 6144); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(46), 6656); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(47), 6144); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(48), 6656); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(49), 6144); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(50), 7168); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(51), 7680); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(52), 7168); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(53), 7680); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(54), 7168); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(55), 6144); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(56), 6656); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(57), 6144); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(58), 6656); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(59), 6144); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(60), 8192); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(61), 8704); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(62), 8192); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(63), 8704); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(64), 8192); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(65), 9216); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(66), 9728); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(67), 9216); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(68), 9728); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(69), 9216); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(70), 8192); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(71), 8704); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(72), 8192); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(73), 8704); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(74), 8192); - EXPECT_INT_EQ(dimensions.chunk_internal_offset(75), 0); + EXPECT_EQ(int, dimensions.chunk_internal_offset(0), 0); + EXPECT_EQ(int, dimensions.chunk_internal_offset(1), 512); + EXPECT_EQ(int, dimensions.chunk_internal_offset(2), 0); + EXPECT_EQ(int, dimensions.chunk_internal_offset(3), 512); + EXPECT_EQ(int, dimensions.chunk_internal_offset(4), 0); + EXPECT_EQ(int, dimensions.chunk_internal_offset(5), 1024); + EXPECT_EQ(int, dimensions.chunk_internal_offset(6), 1536); + EXPECT_EQ(int, dimensions.chunk_internal_offset(7), 1024); + EXPECT_EQ(int, dimensions.chunk_internal_offset(8), 1536); + EXPECT_EQ(int, dimensions.chunk_internal_offset(9), 1024); + EXPECT_EQ(int, dimensions.chunk_internal_offset(10), 0); + EXPECT_EQ(int, dimensions.chunk_internal_offset(11), 512); + EXPECT_EQ(int, dimensions.chunk_internal_offset(12), 0); + EXPECT_EQ(int, dimensions.chunk_internal_offset(13), 512); + EXPECT_EQ(int, dimensions.chunk_internal_offset(14), 0); + EXPECT_EQ(int, dimensions.chunk_internal_offset(15), 2048); + EXPECT_EQ(int, dimensions.chunk_internal_offset(16), 2560); + EXPECT_EQ(int, dimensions.chunk_internal_offset(17), 2048); + EXPECT_EQ(int, dimensions.chunk_internal_offset(18), 2560); + EXPECT_EQ(int, dimensions.chunk_internal_offset(19), 2048); + EXPECT_EQ(int, dimensions.chunk_internal_offset(20), 3072); + EXPECT_EQ(int, dimensions.chunk_internal_offset(21), 3584); + EXPECT_EQ(int, dimensions.chunk_internal_offset(22), 3072); + EXPECT_EQ(int, dimensions.chunk_internal_offset(23), 3584); + EXPECT_EQ(int, dimensions.chunk_internal_offset(24), 3072); + EXPECT_EQ(int, dimensions.chunk_internal_offset(25), 2048); + EXPECT_EQ(int, dimensions.chunk_internal_offset(26), 2560); + EXPECT_EQ(int, dimensions.chunk_internal_offset(27), 2048); + EXPECT_EQ(int, dimensions.chunk_internal_offset(28), 2560); + EXPECT_EQ(int, dimensions.chunk_internal_offset(29), 2048); + EXPECT_EQ(int, dimensions.chunk_internal_offset(30), 4096); + EXPECT_EQ(int, dimensions.chunk_internal_offset(31), 4608); + EXPECT_EQ(int, dimensions.chunk_internal_offset(32), 4096); + EXPECT_EQ(int, dimensions.chunk_internal_offset(33), 4608); + EXPECT_EQ(int, dimensions.chunk_internal_offset(34), 4096); + EXPECT_EQ(int, dimensions.chunk_internal_offset(35), 5120); + EXPECT_EQ(int, dimensions.chunk_internal_offset(36), 5632); + EXPECT_EQ(int, dimensions.chunk_internal_offset(37), 5120); + EXPECT_EQ(int, dimensions.chunk_internal_offset(38), 5632); + EXPECT_EQ(int, dimensions.chunk_internal_offset(39), 5120); + EXPECT_EQ(int, dimensions.chunk_internal_offset(40), 4096); + EXPECT_EQ(int, dimensions.chunk_internal_offset(41), 4608); + EXPECT_EQ(int, dimensions.chunk_internal_offset(42), 4096); + EXPECT_EQ(int, dimensions.chunk_internal_offset(43), 4608); + EXPECT_EQ(int, dimensions.chunk_internal_offset(44), 4096); + EXPECT_EQ(int, dimensions.chunk_internal_offset(45), 6144); + EXPECT_EQ(int, dimensions.chunk_internal_offset(46), 6656); + EXPECT_EQ(int, dimensions.chunk_internal_offset(47), 6144); + EXPECT_EQ(int, dimensions.chunk_internal_offset(48), 6656); + EXPECT_EQ(int, dimensions.chunk_internal_offset(49), 6144); + EXPECT_EQ(int, dimensions.chunk_internal_offset(50), 7168); + EXPECT_EQ(int, dimensions.chunk_internal_offset(51), 7680); + EXPECT_EQ(int, dimensions.chunk_internal_offset(52), 7168); + EXPECT_EQ(int, dimensions.chunk_internal_offset(53), 7680); + EXPECT_EQ(int, dimensions.chunk_internal_offset(54), 7168); + EXPECT_EQ(int, dimensions.chunk_internal_offset(55), 6144); + EXPECT_EQ(int, dimensions.chunk_internal_offset(56), 6656); + EXPECT_EQ(int, dimensions.chunk_internal_offset(57), 6144); + EXPECT_EQ(int, dimensions.chunk_internal_offset(58), 6656); + EXPECT_EQ(int, dimensions.chunk_internal_offset(59), 6144); + EXPECT_EQ(int, dimensions.chunk_internal_offset(60), 8192); + EXPECT_EQ(int, dimensions.chunk_internal_offset(61), 8704); + EXPECT_EQ(int, dimensions.chunk_internal_offset(62), 8192); + EXPECT_EQ(int, dimensions.chunk_internal_offset(63), 8704); + EXPECT_EQ(int, dimensions.chunk_internal_offset(64), 8192); + EXPECT_EQ(int, dimensions.chunk_internal_offset(65), 9216); + EXPECT_EQ(int, dimensions.chunk_internal_offset(66), 9728); + EXPECT_EQ(int, dimensions.chunk_internal_offset(67), 9216); + EXPECT_EQ(int, dimensions.chunk_internal_offset(68), 9728); + EXPECT_EQ(int, dimensions.chunk_internal_offset(69), 9216); + EXPECT_EQ(int, dimensions.chunk_internal_offset(70), 8192); + EXPECT_EQ(int, dimensions.chunk_internal_offset(71), 8704); + EXPECT_EQ(int, dimensions.chunk_internal_offset(72), 8192); + EXPECT_EQ(int, dimensions.chunk_internal_offset(73), 8704); + EXPECT_EQ(int, dimensions.chunk_internal_offset(74), 8192); + EXPECT_EQ(int, dimensions.chunk_internal_offset(75), 0); retval = 0; } catch (const std::exception& exc) { diff --git a/tests/unit-tests/array-dimensions-chunk-lattice-index.cpp b/tests/unit-tests/array-dimensions-chunk-lattice-index.cpp index e37a46d6..7dd48576 100644 --- a/tests/unit-tests/array-dimensions-chunk-lattice-index.cpp +++ b/tests/unit-tests/array-dimensions-chunk-lattice-index.cpp @@ -18,57 +18,57 @@ main() dims.emplace_back("x", ZarrDimensionType_Space, 64, 16, 0); // 4 chunks ArrayDimensions dimensions(std::move(dims), ZarrDataType_uint8); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(0, 2), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(0, 1), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(0, 0), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(1, 2), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(1, 1), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(1, 0), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(2, 2), 1); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(2, 1), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(2, 0), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(3, 2), 1); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(3, 1), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(3, 0), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(4, 2), 2); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(4, 1), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(4, 0), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(5, 2), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(5, 1), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(5, 0), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(12, 2), 1); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(12, 1), 1); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(12, 0), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(19, 2), 2); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(19, 1), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(19, 0), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(26, 2), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(26, 1), 1); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(26, 0), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(33, 2), 1); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(33, 1), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(33, 0), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(40, 2), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(40, 1), 1); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(40, 0), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(47, 2), 1); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(47, 1), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(47, 0), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(54, 2), 2); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(54, 1), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(54, 0), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(61, 2), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(61, 1), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(61, 0), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(68, 2), 1); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(68, 1), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(68, 0), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(74, 2), 2); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(74, 1), 1); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(74, 0), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(75, 2), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(75, 1), 0); - EXPECT_INT_EQ(dimensions.chunk_lattice_index(75, 0), 1); + EXPECT_EQ(int, dimensions.chunk_lattice_index(0, 2), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(0, 1), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(0, 0), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(1, 2), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(1, 1), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(1, 0), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(2, 2), 1); + EXPECT_EQ(int, dimensions.chunk_lattice_index(2, 1), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(2, 0), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(3, 2), 1); + EXPECT_EQ(int, dimensions.chunk_lattice_index(3, 1), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(3, 0), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(4, 2), 2); + EXPECT_EQ(int, dimensions.chunk_lattice_index(4, 1), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(4, 0), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(5, 2), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(5, 1), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(5, 0), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(12, 2), 1); + EXPECT_EQ(int, dimensions.chunk_lattice_index(12, 1), 1); + EXPECT_EQ(int, dimensions.chunk_lattice_index(12, 0), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(19, 2), 2); + EXPECT_EQ(int, dimensions.chunk_lattice_index(19, 1), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(19, 0), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(26, 2), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(26, 1), 1); + EXPECT_EQ(int, dimensions.chunk_lattice_index(26, 0), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(33, 2), 1); + EXPECT_EQ(int, dimensions.chunk_lattice_index(33, 1), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(33, 0), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(40, 2), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(40, 1), 1); + EXPECT_EQ(int, dimensions.chunk_lattice_index(40, 0), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(47, 2), 1); + EXPECT_EQ(int, dimensions.chunk_lattice_index(47, 1), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(47, 0), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(54, 2), 2); + EXPECT_EQ(int, dimensions.chunk_lattice_index(54, 1), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(54, 0), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(61, 2), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(61, 1), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(61, 0), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(68, 2), 1); + EXPECT_EQ(int, dimensions.chunk_lattice_index(68, 1), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(68, 0), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(74, 2), 2); + EXPECT_EQ(int, dimensions.chunk_lattice_index(74, 1), 1); + EXPECT_EQ(int, dimensions.chunk_lattice_index(74, 0), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(75, 2), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(75, 1), 0); + EXPECT_EQ(int, dimensions.chunk_lattice_index(75, 0), 1); retval = 0; } catch (const std::exception& exc) { diff --git a/tests/unit-tests/array-dimensions-shard-index-for-chunk.cpp b/tests/unit-tests/array-dimensions-shard-index-for-chunk.cpp index f9b118c6..71519136 100644 --- a/tests/unit-tests/array-dimensions-shard-index-for-chunk.cpp +++ b/tests/unit-tests/array-dimensions-shard-index-for-chunk.cpp @@ -37,154 +37,154 @@ main() 2); // 4 / 2 = 2 shards ArrayDimensions dimensions(std::move(dims), ZarrDataType_uint32); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(0), 0); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(1), 0); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(2), 1); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(3), 1); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(4), 2); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(5), 2); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(6), 3); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(7), 3); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(8), 4); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(9), 4); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(10), 5); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(11), 5); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(12), 6); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(13), 6); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(14), 7); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(15), 7); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(16), 8); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(17), 8); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(18), 9); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(19), 9); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(20), 10); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(21), 10); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(22), 11); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(23), 11); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(24), 12); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(25), 12); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(26), 13); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(27), 13); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(28), 14); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(29), 14); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(30), 15); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(31), 15); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(32), 16); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(33), 16); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(34), 17); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(35), 17); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(36), 0); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(37), 0); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(38), 1); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(39), 1); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(40), 2); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(41), 2); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(42), 3); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(43), 3); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(44), 4); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(45), 4); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(46), 5); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(47), 5); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(48), 6); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(49), 6); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(50), 7); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(51), 7); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(52), 8); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(53), 8); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(54), 9); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(55), 9); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(56), 10); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(57), 10); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(58), 11); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(59), 11); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(60), 12); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(61), 12); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(62), 13); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(63), 13); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(64), 14); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(65), 14); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(66), 15); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(67), 15); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(68), 16); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(69), 16); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(70), 17); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(71), 17); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(72), 0); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(73), 0); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(74), 1); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(75), 1); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(76), 2); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(77), 2); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(78), 3); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(79), 3); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(80), 4); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(81), 4); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(82), 5); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(83), 5); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(84), 6); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(85), 6); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(86), 7); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(87), 7); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(88), 8); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(89), 8); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(90), 9); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(91), 9); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(92), 10); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(93), 10); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(94), 11); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(95), 11); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(96), 12); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(97), 12); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(98), 13); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(99), 13); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(100), 14); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(101), 14); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(102), 15); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(103), 15); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(104), 16); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(105), 16); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(106), 17); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(107), 17); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(108), 0); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(109), 0); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(110), 1); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(111), 1); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(112), 2); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(113), 2); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(114), 3); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(115), 3); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(116), 4); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(117), 4); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(118), 5); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(119), 5); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(120), 6); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(121), 6); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(122), 7); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(123), 7); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(124), 8); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(125), 8); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(126), 9); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(127), 9); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(128), 10); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(129), 10); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(130), 11); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(131), 11); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(132), 12); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(133), 12); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(134), 13); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(135), 13); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(136), 14); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(137), 14); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(138), 15); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(139), 15); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(140), 16); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(141), 16); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(142), 17); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(143), 17); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(0), 0); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(1), 0); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(2), 1); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(3), 1); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(4), 2); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(5), 2); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(6), 3); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(7), 3); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(8), 4); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(9), 4); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(10), 5); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(11), 5); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(12), 6); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(13), 6); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(14), 7); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(15), 7); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(16), 8); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(17), 8); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(18), 9); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(19), 9); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(20), 10); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(21), 10); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(22), 11); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(23), 11); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(24), 12); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(25), 12); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(26), 13); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(27), 13); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(28), 14); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(29), 14); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(30), 15); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(31), 15); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(32), 16); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(33), 16); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(34), 17); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(35), 17); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(36), 0); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(37), 0); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(38), 1); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(39), 1); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(40), 2); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(41), 2); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(42), 3); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(43), 3); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(44), 4); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(45), 4); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(46), 5); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(47), 5); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(48), 6); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(49), 6); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(50), 7); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(51), 7); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(52), 8); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(53), 8); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(54), 9); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(55), 9); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(56), 10); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(57), 10); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(58), 11); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(59), 11); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(60), 12); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(61), 12); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(62), 13); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(63), 13); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(64), 14); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(65), 14); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(66), 15); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(67), 15); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(68), 16); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(69), 16); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(70), 17); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(71), 17); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(72), 0); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(73), 0); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(74), 1); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(75), 1); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(76), 2); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(77), 2); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(78), 3); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(79), 3); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(80), 4); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(81), 4); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(82), 5); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(83), 5); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(84), 6); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(85), 6); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(86), 7); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(87), 7); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(88), 8); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(89), 8); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(90), 9); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(91), 9); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(92), 10); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(93), 10); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(94), 11); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(95), 11); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(96), 12); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(97), 12); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(98), 13); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(99), 13); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(100), 14); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(101), 14); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(102), 15); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(103), 15); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(104), 16); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(105), 16); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(106), 17); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(107), 17); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(108), 0); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(109), 0); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(110), 1); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(111), 1); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(112), 2); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(113), 2); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(114), 3); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(115), 3); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(116), 4); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(117), 4); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(118), 5); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(119), 5); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(120), 6); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(121), 6); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(122), 7); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(123), 7); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(124), 8); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(125), 8); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(126), 9); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(127), 9); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(128), 10); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(129), 10); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(130), 11); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(131), 11); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(132), 12); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(133), 12); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(134), 13); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(135), 13); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(136), 14); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(137), 14); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(138), 15); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(139), 15); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(140), 16); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(141), 16); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(142), 17); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(143), 17); retval = 0; } catch (const std::exception& exc) { - LOG_ERROR("Exception: %s\n", exc.what()); + LOG_ERROR("Exception: ", exc.what()); } catch (...) { LOG_ERROR("Exception: (unknown)"); } diff --git a/tests/unit-tests/array-dimensions-shard-internal-index.cpp b/tests/unit-tests/array-dimensions-shard-internal-index.cpp index 41ce9b75..73353be4 100644 --- a/tests/unit-tests/array-dimensions-shard-internal-index.cpp +++ b/tests/unit-tests/array-dimensions-shard-internal-index.cpp @@ -27,41 +27,41 @@ main() ArrayDimensions dimensions(std::move(dims), ZarrDataType_uint64); try { - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(0), 0); - EXPECT_INT_EQ(dimensions.shard_internal_index(0), 0); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(0), 0); + EXPECT_EQ(int, dimensions.shard_internal_index(0), 0); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(1), 0); - EXPECT_INT_EQ(dimensions.shard_internal_index(1), 1); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(1), 0); + EXPECT_EQ(int, dimensions.shard_internal_index(1), 1); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(2), 0); - EXPECT_INT_EQ(dimensions.shard_internal_index(2), 2); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(2), 0); + EXPECT_EQ(int, dimensions.shard_internal_index(2), 2); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(3), 1); - EXPECT_INT_EQ(dimensions.shard_internal_index(3), 0); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(3), 1); + EXPECT_EQ(int, dimensions.shard_internal_index(3), 0); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(4), 0); - EXPECT_INT_EQ(dimensions.shard_internal_index(4), 3); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(4), 0); + EXPECT_EQ(int, dimensions.shard_internal_index(4), 3); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(5), 0); - EXPECT_INT_EQ(dimensions.shard_internal_index(5), 4); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(5), 0); + EXPECT_EQ(int, dimensions.shard_internal_index(5), 4); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(6), 0); - EXPECT_INT_EQ(dimensions.shard_internal_index(6), 5); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(6), 0); + EXPECT_EQ(int, dimensions.shard_internal_index(6), 5); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(7), 1); - EXPECT_INT_EQ(dimensions.shard_internal_index(7), 3); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(7), 1); + EXPECT_EQ(int, dimensions.shard_internal_index(7), 3); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(8), 2); - EXPECT_INT_EQ(dimensions.shard_internal_index(8), 0); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(8), 2); + EXPECT_EQ(int, dimensions.shard_internal_index(8), 0); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(9), 2); - EXPECT_INT_EQ(dimensions.shard_internal_index(9), 1); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(9), 2); + EXPECT_EQ(int, dimensions.shard_internal_index(9), 1); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(10), 2); - EXPECT_INT_EQ(dimensions.shard_internal_index(10), 2); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(10), 2); + EXPECT_EQ(int, dimensions.shard_internal_index(10), 2); - EXPECT_INT_EQ(dimensions.shard_index_for_chunk(11), 3); - EXPECT_INT_EQ(dimensions.shard_internal_index(11), 0); + EXPECT_EQ(int, dimensions.shard_index_for_chunk(11), 3); + EXPECT_EQ(int, dimensions.shard_internal_index(11), 0); retval = 0; } catch (const std::exception& exc) { LOG_ERROR("Exception: ", exc.what()); diff --git a/tests/unit-tests/array-dimensions-tile-group-offset.cpp b/tests/unit-tests/array-dimensions-tile-group-offset.cpp index 22f0a5e7..f4714af8 100644 --- a/tests/unit-tests/array-dimensions-tile-group-offset.cpp +++ b/tests/unit-tests/array-dimensions-tile-group-offset.cpp @@ -18,86 +18,86 @@ main() ArrayDimensions dimensions(std::move(dims), ZarrDataType_float32); try { - EXPECT_INT_EQ(dimensions.tile_group_offset(0), 0); - EXPECT_INT_EQ(dimensions.tile_group_offset(1), 0); - EXPECT_INT_EQ(dimensions.tile_group_offset(2), 12); - EXPECT_INT_EQ(dimensions.tile_group_offset(3), 12); - EXPECT_INT_EQ(dimensions.tile_group_offset(4), 24); - EXPECT_INT_EQ(dimensions.tile_group_offset(5), 0); - EXPECT_INT_EQ(dimensions.tile_group_offset(6), 0); - EXPECT_INT_EQ(dimensions.tile_group_offset(7), 12); - EXPECT_INT_EQ(dimensions.tile_group_offset(8), 12); - EXPECT_INT_EQ(dimensions.tile_group_offset(9), 24); - EXPECT_INT_EQ(dimensions.tile_group_offset(10), 36); - EXPECT_INT_EQ(dimensions.tile_group_offset(11), 36); - EXPECT_INT_EQ(dimensions.tile_group_offset(12), 48); - EXPECT_INT_EQ(dimensions.tile_group_offset(13), 48); - EXPECT_INT_EQ(dimensions.tile_group_offset(14), 60); - EXPECT_INT_EQ(dimensions.tile_group_offset(15), 0); - EXPECT_INT_EQ(dimensions.tile_group_offset(16), 0); - EXPECT_INT_EQ(dimensions.tile_group_offset(17), 12); - EXPECT_INT_EQ(dimensions.tile_group_offset(18), 12); - EXPECT_INT_EQ(dimensions.tile_group_offset(19), 24); - EXPECT_INT_EQ(dimensions.tile_group_offset(20), 0); - EXPECT_INT_EQ(dimensions.tile_group_offset(21), 0); - EXPECT_INT_EQ(dimensions.tile_group_offset(22), 12); - EXPECT_INT_EQ(dimensions.tile_group_offset(23), 12); - EXPECT_INT_EQ(dimensions.tile_group_offset(24), 24); - EXPECT_INT_EQ(dimensions.tile_group_offset(25), 36); - EXPECT_INT_EQ(dimensions.tile_group_offset(26), 36); - EXPECT_INT_EQ(dimensions.tile_group_offset(27), 48); - EXPECT_INT_EQ(dimensions.tile_group_offset(28), 48); - EXPECT_INT_EQ(dimensions.tile_group_offset(29), 60); - EXPECT_INT_EQ(dimensions.tile_group_offset(30), 0); - EXPECT_INT_EQ(dimensions.tile_group_offset(31), 0); - EXPECT_INT_EQ(dimensions.tile_group_offset(32), 12); - EXPECT_INT_EQ(dimensions.tile_group_offset(33), 12); - EXPECT_INT_EQ(dimensions.tile_group_offset(34), 24); - EXPECT_INT_EQ(dimensions.tile_group_offset(35), 0); - EXPECT_INT_EQ(dimensions.tile_group_offset(36), 0); - EXPECT_INT_EQ(dimensions.tile_group_offset(37), 12); - EXPECT_INT_EQ(dimensions.tile_group_offset(38), 12); - EXPECT_INT_EQ(dimensions.tile_group_offset(39), 24); - EXPECT_INT_EQ(dimensions.tile_group_offset(40), 36); - EXPECT_INT_EQ(dimensions.tile_group_offset(41), 36); - EXPECT_INT_EQ(dimensions.tile_group_offset(42), 48); - EXPECT_INT_EQ(dimensions.tile_group_offset(43), 48); - EXPECT_INT_EQ(dimensions.tile_group_offset(44), 60); - EXPECT_INT_EQ(dimensions.tile_group_offset(45), 0); - EXPECT_INT_EQ(dimensions.tile_group_offset(46), 0); - EXPECT_INT_EQ(dimensions.tile_group_offset(47), 12); - EXPECT_INT_EQ(dimensions.tile_group_offset(48), 12); - EXPECT_INT_EQ(dimensions.tile_group_offset(49), 24); - EXPECT_INT_EQ(dimensions.tile_group_offset(50), 0); - EXPECT_INT_EQ(dimensions.tile_group_offset(51), 0); - EXPECT_INT_EQ(dimensions.tile_group_offset(52), 12); - EXPECT_INT_EQ(dimensions.tile_group_offset(53), 12); - EXPECT_INT_EQ(dimensions.tile_group_offset(54), 24); - EXPECT_INT_EQ(dimensions.tile_group_offset(55), 36); - EXPECT_INT_EQ(dimensions.tile_group_offset(56), 36); - EXPECT_INT_EQ(dimensions.tile_group_offset(57), 48); - EXPECT_INT_EQ(dimensions.tile_group_offset(58), 48); - EXPECT_INT_EQ(dimensions.tile_group_offset(59), 60); - EXPECT_INT_EQ(dimensions.tile_group_offset(60), 0); - EXPECT_INT_EQ(dimensions.tile_group_offset(61), 0); - EXPECT_INT_EQ(dimensions.tile_group_offset(62), 12); - EXPECT_INT_EQ(dimensions.tile_group_offset(63), 12); - EXPECT_INT_EQ(dimensions.tile_group_offset(64), 24); - EXPECT_INT_EQ(dimensions.tile_group_offset(65), 0); - EXPECT_INT_EQ(dimensions.tile_group_offset(66), 0); - EXPECT_INT_EQ(dimensions.tile_group_offset(67), 12); - EXPECT_INT_EQ(dimensions.tile_group_offset(68), 12); - EXPECT_INT_EQ(dimensions.tile_group_offset(69), 24); - EXPECT_INT_EQ(dimensions.tile_group_offset(70), 36); - EXPECT_INT_EQ(dimensions.tile_group_offset(71), 36); - EXPECT_INT_EQ(dimensions.tile_group_offset(72), 48); - EXPECT_INT_EQ(dimensions.tile_group_offset(73), 48); - EXPECT_INT_EQ(dimensions.tile_group_offset(74), 60); - EXPECT_INT_EQ(dimensions.tile_group_offset(75), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(0), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(1), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(2), 12); + EXPECT_EQ(int, dimensions.tile_group_offset(3), 12); + EXPECT_EQ(int, dimensions.tile_group_offset(4), 24); + EXPECT_EQ(int, dimensions.tile_group_offset(5), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(6), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(7), 12); + EXPECT_EQ(int, dimensions.tile_group_offset(8), 12); + EXPECT_EQ(int, dimensions.tile_group_offset(9), 24); + EXPECT_EQ(int, dimensions.tile_group_offset(10), 36); + EXPECT_EQ(int, dimensions.tile_group_offset(11), 36); + EXPECT_EQ(int, dimensions.tile_group_offset(12), 48); + EXPECT_EQ(int, dimensions.tile_group_offset(13), 48); + EXPECT_EQ(int, dimensions.tile_group_offset(14), 60); + EXPECT_EQ(int, dimensions.tile_group_offset(15), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(16), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(17), 12); + EXPECT_EQ(int, dimensions.tile_group_offset(18), 12); + EXPECT_EQ(int, dimensions.tile_group_offset(19), 24); + EXPECT_EQ(int, dimensions.tile_group_offset(20), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(21), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(22), 12); + EXPECT_EQ(int, dimensions.tile_group_offset(23), 12); + EXPECT_EQ(int, dimensions.tile_group_offset(24), 24); + EXPECT_EQ(int, dimensions.tile_group_offset(25), 36); + EXPECT_EQ(int, dimensions.tile_group_offset(26), 36); + EXPECT_EQ(int, dimensions.tile_group_offset(27), 48); + EXPECT_EQ(int, dimensions.tile_group_offset(28), 48); + EXPECT_EQ(int, dimensions.tile_group_offset(29), 60); + EXPECT_EQ(int, dimensions.tile_group_offset(30), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(31), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(32), 12); + EXPECT_EQ(int, dimensions.tile_group_offset(33), 12); + EXPECT_EQ(int, dimensions.tile_group_offset(34), 24); + EXPECT_EQ(int, dimensions.tile_group_offset(35), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(36), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(37), 12); + EXPECT_EQ(int, dimensions.tile_group_offset(38), 12); + EXPECT_EQ(int, dimensions.tile_group_offset(39), 24); + EXPECT_EQ(int, dimensions.tile_group_offset(40), 36); + EXPECT_EQ(int, dimensions.tile_group_offset(41), 36); + EXPECT_EQ(int, dimensions.tile_group_offset(42), 48); + EXPECT_EQ(int, dimensions.tile_group_offset(43), 48); + EXPECT_EQ(int, dimensions.tile_group_offset(44), 60); + EXPECT_EQ(int, dimensions.tile_group_offset(45), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(46), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(47), 12); + EXPECT_EQ(int, dimensions.tile_group_offset(48), 12); + EXPECT_EQ(int, dimensions.tile_group_offset(49), 24); + EXPECT_EQ(int, dimensions.tile_group_offset(50), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(51), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(52), 12); + EXPECT_EQ(int, dimensions.tile_group_offset(53), 12); + EXPECT_EQ(int, dimensions.tile_group_offset(54), 24); + EXPECT_EQ(int, dimensions.tile_group_offset(55), 36); + EXPECT_EQ(int, dimensions.tile_group_offset(56), 36); + EXPECT_EQ(int, dimensions.tile_group_offset(57), 48); + EXPECT_EQ(int, dimensions.tile_group_offset(58), 48); + EXPECT_EQ(int, dimensions.tile_group_offset(59), 60); + EXPECT_EQ(int, dimensions.tile_group_offset(60), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(61), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(62), 12); + EXPECT_EQ(int, dimensions.tile_group_offset(63), 12); + EXPECT_EQ(int, dimensions.tile_group_offset(64), 24); + EXPECT_EQ(int, dimensions.tile_group_offset(65), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(66), 0); + EXPECT_EQ(int, dimensions.tile_group_offset(67), 12); + EXPECT_EQ(int, dimensions.tile_group_offset(68), 12); + EXPECT_EQ(int, dimensions.tile_group_offset(69), 24); + EXPECT_EQ(int, dimensions.tile_group_offset(70), 36); + EXPECT_EQ(int, dimensions.tile_group_offset(71), 36); + EXPECT_EQ(int, dimensions.tile_group_offset(72), 48); + EXPECT_EQ(int, dimensions.tile_group_offset(73), 48); + EXPECT_EQ(int, dimensions.tile_group_offset(74), 60); + EXPECT_EQ(int, dimensions.tile_group_offset(75), 0); retval = 0; } catch (const std::exception& exc) { - LOG_ERROR("Exception: %s\n", exc.what()); + LOG_ERROR("Exception: ", exc.what()); } catch (...) { LOG_ERROR("Exception: (unknown)"); } diff --git a/tests/unit-tests/create-stream.cpp b/tests/unit-tests/create-stream.cpp index c66bfacf..468e37e5 100644 --- a/tests/unit-tests/create-stream.cpp +++ b/tests/unit-tests/create-stream.cpp @@ -65,7 +65,7 @@ main() retval = 0; } catch (const std::exception& exception) { - LOG_ERROR("%s", exception.what()); + LOG_ERROR(exception.what()); } // cleanup diff --git a/tests/unit-tests/file-sink-write.cpp b/tests/unit-tests/file-sink-write.cpp new file mode 100644 index 00000000..9df1e8f4 --- /dev/null +++ b/tests/unit-tests/file-sink-write.cpp @@ -0,0 +1,53 @@ +#include "file.sink.hh" +#include "unit.test.macros.hh" + +#include +#include +#include + +namespace fs = std::filesystem; + +int +main() +{ + int retval = 0; + fs::path tmp_path = fs::temp_directory_path() / TEST; + + try { + CHECK(!fs::exists(tmp_path)); + { + char str[] = "Hello, Acquire!"; + auto sink = std::make_unique(tmp_path.string()); + + std::span data = { reinterpret_cast(str), + sizeof(str) - 1 }; + CHECK(sink->write(0, data)); + CHECK(zarr::finalize_sink(std::move(sink))); + } + + // The file tmp_path should now contain the string "Hello, world!\n". + CHECK(fs::exists(tmp_path)); + + std::ifstream ifs(tmp_path); + CHECK(ifs.is_open()); + + std::string contents; + while (!ifs.eof()) { + std::getline(ifs, contents); + } + ifs.close(); + + EXPECT_STR_EQ(contents.c_str(), "Hello, Acquire!"); + } catch (const std::exception& e) { + LOG_ERROR("Caught exception: ", e.what()); + retval = 1; + } + + std::error_code ec; + if (!fs::remove(tmp_path, ec)) { + LOG_ERROR("Failed to remove file: ", ec.message()); + retval = 1; + } + + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/s3-connection-bucket-exists.cpp b/tests/unit-tests/s3-connection-bucket-exists.cpp index b62c25a1..86047471 100644 --- a/tests/unit-tests/s3-connection-bucket-exists.cpp +++ b/tests/unit-tests/s3-connection-bucket-exists.cpp @@ -72,7 +72,7 @@ main() retval = 0; } catch (const std::exception& e) { - LOG_ERROR("Failed: %s", e.what()); + LOG_ERROR("Failed: ", e.what()); } return retval; diff --git a/tests/unit-tests/s3-connection-object-exists-check-false-positives.cpp b/tests/unit-tests/s3-connection-object-exists-check-false-positives.cpp index 944ae327..b2afd643 100644 --- a/tests/unit-tests/s3-connection-object-exists-check-false-positives.cpp +++ b/tests/unit-tests/s3-connection-object-exists-check-false-positives.cpp @@ -78,7 +78,7 @@ main() retval = 0; } catch (const std::exception& e) { - LOG_ERROR("Failed: %s", e.what()); + LOG_ERROR("Failed: ", e.what()); } return retval; diff --git a/tests/unit-tests/s3-connection-put-object.cpp b/tests/unit-tests/s3-connection-put-object.cpp index beef999b..678fdf8a 100644 --- a/tests/unit-tests/s3-connection-put-object.cpp +++ b/tests/unit-tests/s3-connection-put-object.cpp @@ -81,7 +81,7 @@ main() retval = 0; } catch (const std::exception& e) { - LOG_ERROR("Failed: %s", e.what()); + LOG_ERROR("Failed: ", e.what()); } return retval; diff --git a/tests/unit-tests/s3-connection-upload-multipart-object.cpp b/tests/unit-tests/s3-connection-upload-multipart-object.cpp index d3d6b316..684569b8 100644 --- a/tests/unit-tests/s3-connection-upload-multipart-object.cpp +++ b/tests/unit-tests/s3-connection-upload-multipart-object.cpp @@ -120,7 +120,7 @@ main() retval = 0; } catch (const std::exception& e) { - LOG_ERROR("Failed: %s", e.what()); + LOG_ERROR("Failed: ", e.what()); } return retval; diff --git a/tests/unit-tests/s3-sink-write.cpp b/tests/unit-tests/s3-sink-write.cpp new file mode 100644 index 00000000..68326163 --- /dev/null +++ b/tests/unit-tests/s3-sink-write.cpp @@ -0,0 +1,127 @@ +#include "s3.sink.hh" +#include "unit.test.macros.hh" + +#include +#include + +namespace { +bool +get_credentials(std::string& endpoint, + std::string& bucket_name, + std::string& access_key_id, + std::string& secret_access_key) +{ + char* env = nullptr; + if (!(env = std::getenv("ZARR_S3_ENDPOINT"))) { + LOG_ERROR("ZARR_S3_ENDPOINT not set."); + return false; + } + endpoint = env; + + if (!(env = std::getenv("ZARR_S3_BUCKET_NAME"))) { + LOG_ERROR("ZARR_S3_BUCKET_NAME not set."); + return false; + } + bucket_name = env; + + if (!(env = std::getenv("ZARR_S3_ACCESS_KEY_ID"))) { + LOG_ERROR("ZARR_S3_ACCESS_KEY_ID not set."); + return false; + } + access_key_id = env; + + if (!(env = std::getenv("ZARR_S3_SECRET_ACCESS_KEY"))) { + LOG_ERROR("ZARR_S3_SECRET_ACCESS_KEY not set."); + return false; + } + secret_access_key = env; + + return true; +} +} // namespace + +int +main() +{ + std::string s3_endpoint, bucket_name, s3_access_key_id, + s3_secret_access_key; + + if (!get_credentials( + s3_endpoint, bucket_name, s3_access_key_id, s3_secret_access_key)) { + LOG_WARNING("Failed to get credentials. Skipping test."); + return 0; + } + + int retval = 1; + const std::string object_name = "test-object"; + + try { + auto pool = std::make_shared( + 1, s3_endpoint, s3_access_key_id, s3_secret_access_key); + + auto conn = pool->get_connection(); + if (!conn->is_connection_valid()) { + LOG_ERROR("Failed to connect to S3."); + return 1; + } + CHECK(conn->bucket_exists(bucket_name)); + CHECK(conn->delete_object(bucket_name, object_name)); + CHECK(!conn->object_exists(bucket_name, object_name)); + + pool->return_connection(std::move(conn)); + + { + char str[] = "Hello, Acquire!"; + auto sink = + std::make_unique(bucket_name, object_name, pool); + std::span data{ reinterpret_cast(str), + sizeof(str) - 1 }; + CHECK(sink->write(0, data)); + CHECK(zarr::finalize_sink(std::move(sink))); + } + + conn = pool->get_connection(); + CHECK(conn->object_exists(bucket_name, object_name)); + pool->return_connection(std::move(conn)); + + // Verify the object contents. + { + minio::s3::BaseUrl url(s3_endpoint); + url.https = s3_endpoint.starts_with("https://"); + + minio::creds::StaticProvider provider(s3_access_key_id, + s3_secret_access_key); + + minio::s3::Client client(url, &provider); + minio::s3::GetObjectArgs args; + args.bucket = bucket_name; + args.object = object_name; + + std::string contents; + args.datafunc = + [&contents](minio::http::DataFunctionArgs args) -> bool { + contents = args.datachunk; + return true; + }; + + // Call get object. + minio::s3::GetObjectResponse resp = client.GetObject(args); + + if (contents != "Hello, Acquire!") { + LOG_ERROR( + "Expected 'Hello, Acquire!' but got '", contents, "'"); + return 1; + } + } + + // cleanup + conn = pool->get_connection(); + CHECK(conn->delete_object(bucket_name, object_name)); + + retval = 0; + } catch (const std::exception& e) { + LOG_ERROR("Exception: ", e.what()); + } + + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/sink-creator-make-data-sinks.cpp b/tests/unit-tests/sink-creator-make-data-sinks.cpp new file mode 100644 index 00000000..09cee1e9 --- /dev/null +++ b/tests/unit-tests/sink-creator-make-data-sinks.cpp @@ -0,0 +1,280 @@ +#include "sink.creator.hh" +#include "s3.connection.hh" +#include "zarr.common.hh" +#include "acquire.zarr.h" +#include "unit.test.macros.hh" + +#include +#include + +namespace fs = std::filesystem; + +namespace { +const std::string test_dir = TEST "-data"; + +bool +get_credentials(std::string& endpoint, + std::string& bucket_name, + std::string& access_key_id, + std::string& secret_access_key) +{ + char* env = nullptr; + if (!(env = std::getenv("ZARR_S3_ENDPOINT"))) { + LOG_ERROR("ZARR_S3_ENDPOINT not set."); + return false; + } + endpoint = env; + + if (!(env = std::getenv("ZARR_S3_BUCKET_NAME"))) { + LOG_ERROR("ZARR_S3_BUCKET_NAME not set."); + return false; + } + bucket_name = env; + + if (!(env = std::getenv("ZARR_S3_ACCESS_KEY_ID"))) { + LOG_ERROR("ZARR_S3_ACCESS_KEY_ID not set."); + return false; + } + access_key_id = env; + + if (!(env = std::getenv("ZARR_S3_SECRET_ACCESS_KEY"))) { + LOG_ERROR("ZARR_S3_SECRET_ACCESS_KEY not set."); + return false; + } + secret_access_key = env; + + return true; +} +} // namespace + +void +sink_creator_make_chunk_sinks(std::shared_ptr thread_pool, + std::shared_ptr dimensions) +{ + zarr::SinkCreator sink_creator(thread_pool, nullptr); + + // create the sinks, then let them go out of scope to close the handles + { + std::vector> sinks; + CHECK(sink_creator.make_data_sinks( + test_dir, dimensions, zarr::chunks_along_dimension, sinks)); + } + + const auto chunks_in_y = zarr::chunks_along_dimension(dimensions->height_dim()); + const auto chunks_in_x = zarr::chunks_along_dimension(dimensions->width_dim()); + + const fs::path base_path(test_dir); + for (auto i = 0; i < chunks_in_y; ++i) { + const fs::path y_dir = base_path / std::to_string(i); + + for (auto j = 0; j < chunks_in_x; ++j) { + const fs::path x_file = y_dir / std::to_string(j); + CHECK(fs::is_regular_file(x_file)); + + // cleanup + fs::remove(x_file); + } + CHECK(!fs::is_regular_file(y_dir / std::to_string(chunks_in_x))); + fs::remove(y_dir); + } + CHECK(!fs::is_directory(base_path / std::to_string(chunks_in_y))); +} + +void +sink_creator_make_chunk_sinks( + std::shared_ptr thread_pool, + std::shared_ptr connection_pool, + const std::string& bucket_name, + std::shared_ptr dimensions) +{ + zarr::SinkCreator sink_creator(thread_pool, connection_pool); + + // create the sinks, then let them go out of scope to close the handles + { + char data_[] = { 0, 0 }; + std::span data(reinterpret_cast(data_), sizeof(data_)); + std::vector> sinks; + CHECK(sink_creator.make_data_sinks(bucket_name, + test_dir, + dimensions, + zarr::chunks_along_dimension, + sinks)); + + for (auto& sink : sinks) { + CHECK(sink); + // we need to write some data to the sink to ensure it is created + CHECK(sink->write(0, data)); + CHECK(zarr::finalize_sink(std::move(sink))); + } + } + + const auto chunks_in_y = zarr::chunks_along_dimension(dimensions->height_dim()); + const auto chunks_in_x = zarr::chunks_along_dimension(dimensions->width_dim()); + + auto conn = connection_pool->get_connection(); + + const std::string base_path(test_dir); + for (auto i = 0; i < chunks_in_y; ++i) { + const std::string y_dir = base_path + "/" + std::to_string(i); + + for (auto j = 0; j < chunks_in_x; ++j) { + const std::string x_file = y_dir + "/" + std::to_string(j); + CHECK(conn->object_exists(bucket_name, x_file)); + + // cleanup + CHECK(conn->delete_object(bucket_name, x_file)); + } + CHECK(!conn->object_exists(bucket_name, + y_dir + "/" + std::to_string(chunks_in_x))); + CHECK(conn->delete_object(bucket_name, y_dir)); + } + CHECK(!conn->object_exists(bucket_name, + base_path + "/" + std::to_string(chunks_in_y))); + CHECK(conn->delete_object(bucket_name, base_path)); +} + +void +sink_creator_make_shard_sinks(std::shared_ptr thread_pool, + std::shared_ptr dimensions) +{ + zarr::SinkCreator sink_creator(thread_pool, nullptr); + + // create the sinks, then let them go out of scope to close the handles + { + std::vector> sinks; + CHECK(sink_creator.make_data_sinks( + test_dir, dimensions, zarr::shards_along_dimension, sinks)); + } + + const auto shards_in_y = zarr::shards_along_dimension(dimensions->height_dim()); + const auto shards_in_x = zarr::shards_along_dimension(dimensions->width_dim()); + + const fs::path base_path(test_dir); + for (auto i = 0; i < shards_in_y; ++i) { + const fs::path y_dir = base_path / std::to_string(i); + + for (auto j = 0; j < shards_in_x; ++j) { + const fs::path x_file = y_dir / std::to_string(j); + CHECK(fs::is_regular_file(x_file)); + + // cleanup + fs::remove(x_file); + } + CHECK(!fs::is_regular_file(y_dir / std::to_string(shards_in_x))); + fs::remove(y_dir); + } + CHECK(!fs::is_directory(base_path / std::to_string(shards_in_y))); +} + +void +sink_creator_make_shard_sinks( + std::shared_ptr thread_pool, + std::shared_ptr connection_pool, + const std::string& bucket_name, + std::shared_ptr dimensions) +{ + zarr::SinkCreator sink_creator(thread_pool, connection_pool); + + // create the sinks, then let them go out of scope to close the handles + { + char data_[] = { 0, 0 }; + std::span data(reinterpret_cast(data_), sizeof(data_)); + std::vector> sinks; + CHECK(sink_creator.make_data_sinks(bucket_name, + test_dir, + dimensions, + zarr::shards_along_dimension, + sinks)); + + for (auto& sink : sinks) { + CHECK(sink); + // we need to write some data to the sink to ensure it is created + CHECK(sink->write(0, data)); + CHECK(zarr::finalize_sink(std::move(sink))); + } + } + + const auto shards_in_y = zarr::shards_along_dimension(dimensions->height_dim()); + const auto shards_in_x = zarr::shards_along_dimension(dimensions->width_dim()); + + auto conn = connection_pool->get_connection(); + + const std::string base_path(test_dir); + for (auto i = 0; i < shards_in_y; ++i) { + const std::string y_dir = base_path + "/" + std::to_string(i); + + for (auto j = 0; j < shards_in_x; ++j) { + const std::string x_file = y_dir + "/" + std::to_string(j); + CHECK(conn->object_exists(bucket_name, x_file)); + + // cleanup + CHECK(conn->delete_object(bucket_name, x_file)); + } + CHECK(!conn->object_exists(bucket_name, + y_dir + "/" + std::to_string(shards_in_x))); + CHECK(conn->delete_object(bucket_name, y_dir)); + } + CHECK(!conn->object_exists(bucket_name, + base_path + "/" + std::to_string(shards_in_y))); + CHECK(conn->delete_object(bucket_name, base_path)); +} + +int +main() +{ + Logger::set_log_level(LogLevel_Debug); + + std::vector dims; + dims.emplace_back("z", + ZarrDimensionType_Space, + 0, + 3, // 3 planes per chunk + 1); // 1 chunk per shard (3 planes per shard) + dims.emplace_back("y", + ZarrDimensionType_Space, + 4, + 2, // 2 rows per chunk, 2 chunks + 2); // 2 chunks per shard (4 rows per shard, 1 shard) + dims.emplace_back("x", + ZarrDimensionType_Space, + 12, + 3, // 3 columns per chunk, 4 chunks + 2); // 2 chunks per shard (6 columns per shard, 2 shards) + auto dimensions = + std::make_shared(std::move(dims), ZarrDataType_int8); + + auto thread_pool = std::make_shared( + std::thread::hardware_concurrency(), + [](const std::string& err) { LOG_ERROR("Failed: ", err.c_str()); }); + + try { + sink_creator_make_chunk_sinks(thread_pool, dimensions); + sink_creator_make_shard_sinks(thread_pool, dimensions); + } catch (const std::exception& e) { + LOG_ERROR("Failed: ", e.what()); + return 1; + } + + std::string s3_endpoint, bucket_name, s3_access_key_id, + s3_secret_access_key; + if (!get_credentials( + s3_endpoint, bucket_name, s3_access_key_id, s3_secret_access_key)) { + LOG_WARNING("Failed to get credentials. Skipping S3 portion of test."); + return 0; + } + + auto connection_pool = std::make_shared( + 4, s3_endpoint, s3_access_key_id, s3_secret_access_key); + + try { + sink_creator_make_chunk_sinks( + thread_pool, connection_pool, bucket_name, dimensions); + sink_creator_make_shard_sinks( + thread_pool, connection_pool, bucket_name, dimensions); + } catch (const std::exception& e) { + LOG_ERROR("Failed: ", e.what()); + return 1; + } + + return 0; +} \ No newline at end of file diff --git a/tests/unit-tests/sink-creator-make-metadata-sinks.cpp b/tests/unit-tests/sink-creator-make-metadata-sinks.cpp new file mode 100644 index 00000000..c4000797 --- /dev/null +++ b/tests/unit-tests/sink-creator-make-metadata-sinks.cpp @@ -0,0 +1,216 @@ +#include "sink.creator.hh" +#include "s3.connection.hh" +#include "unit.test.macros.hh" + +#include +#include + +namespace fs = std::filesystem; + +namespace { +const std::string test_dir = TEST "-data"; + +bool +get_credentials(std::string& endpoint, + std::string& bucket_name, + std::string& access_key_id, + std::string& secret_access_key) +{ + char* env = nullptr; + if (!(env = std::getenv("ZARR_S3_ENDPOINT"))) { + LOG_ERROR("ZARR_S3_ENDPOINT not set."); + return false; + } + endpoint = env; + + if (!(env = std::getenv("ZARR_S3_BUCKET_NAME"))) { + LOG_ERROR("ZARR_S3_BUCKET_NAME not set."); + return false; + } + bucket_name = env; + + if (!(env = std::getenv("ZARR_S3_ACCESS_KEY_ID"))) { + LOG_ERROR("ZARR_S3_ACCESS_KEY_ID not set."); + return false; + } + access_key_id = env; + + if (!(env = std::getenv("ZARR_S3_SECRET_ACCESS_KEY"))) { + LOG_ERROR("ZARR_S3_SECRET_ACCESS_KEY not set."); + return false; + } + secret_access_key = env; + + return true; +} +} // namespace + +void +sink_creator_make_v2_metadata_sinks( + std::shared_ptr thread_pool) +{ + zarr::SinkCreator sink_creator(thread_pool, nullptr); + + std::unordered_map> metadata_sinks; + CHECK(sink_creator.make_metadata_sinks(2, test_dir, metadata_sinks)); + + CHECK(metadata_sinks.size() == 4); + CHECK(metadata_sinks.contains(".zattrs")); + CHECK(metadata_sinks.contains(".zgroup")); + CHECK(metadata_sinks.contains("0/.zattrs")); + CHECK(metadata_sinks.contains("acquire.json")); + + for (auto& [key, sink] : metadata_sinks) { + CHECK(sink); + sink.reset(nullptr); // close the file + + fs::path file_path(test_dir + "/" + key); + CHECK(fs::is_regular_file(file_path)); + // cleanup + fs::remove(file_path); + } + + fs::remove(test_dir + "/0"); +} + +void +sink_creator_make_v2_metadata_sinks( + std::shared_ptr thread_pool, + std::shared_ptr connection_pool, + const std::string& bucket_name) +{ + zarr::SinkCreator sink_creator(thread_pool, connection_pool); + + std::unordered_map> metadata_sinks; + CHECK( + sink_creator.make_metadata_sinks(2, bucket_name, test_dir, metadata_sinks)); + + CHECK(metadata_sinks.size() == 4); + CHECK(metadata_sinks.contains(".zattrs")); + CHECK(metadata_sinks.contains(".zgroup")); + CHECK(metadata_sinks.contains("0/.zattrs")); + CHECK(metadata_sinks.contains("acquire.json")); + + auto conn = connection_pool->get_connection(); + + char data_[] = { 0, 0 }; + std::span data(reinterpret_cast(data_), sizeof(data_)); + for (auto& [key, sink] : metadata_sinks) { + CHECK(sink); + // we need to write some data to the sink to ensure it is created + CHECK(sink->write(0, data)); + + CHECK(zarr::finalize_sink(std::move(sink))); // close the connection + + std::string path = test_dir + "/" + key; + CHECK(conn->object_exists(bucket_name, path)); + // cleanup + CHECK(conn->delete_object(bucket_name, path)); + } + + CHECK(conn->delete_object(bucket_name, "0")); +} + +void +sink_creator_make_v3_metadata_sinks( + std::shared_ptr thread_pool) +{ + zarr::SinkCreator sink_creator(thread_pool, nullptr); + + std::unordered_map> metadata_sinks; + CHECK(sink_creator.make_metadata_sinks(3, test_dir, metadata_sinks)); + + CHECK(metadata_sinks.size() == 3); + CHECK(metadata_sinks.contains("zarr.json")); + CHECK(metadata_sinks.contains("meta/root.group.json")); + CHECK(metadata_sinks.contains("meta/acquire.json")); + + for (auto& [key, sink] : metadata_sinks) { + CHECK(sink); + sink.reset(nullptr); // close the file + + fs::path file_path(test_dir + "/" + key); + CHECK(fs::is_regular_file(file_path)); + // cleanup + fs::remove(file_path); + } + + fs::remove(test_dir + "/meta"); +} + +void +sink_creator_make_v3_metadata_sinks( + std::shared_ptr thread_pool, + std::shared_ptr connection_pool, + const std::string& bucket_name) +{ + zarr::SinkCreator sink_creator(thread_pool, connection_pool); + + std::unordered_map> metadata_sinks; + CHECK( + sink_creator.make_metadata_sinks(3, bucket_name, test_dir, metadata_sinks)); + + CHECK(metadata_sinks.size() == 3); + CHECK(metadata_sinks.contains("zarr.json")); + CHECK(metadata_sinks.contains("meta/root.group.json")); + CHECK(metadata_sinks.contains("meta/acquire.json")); + + auto conn = connection_pool->get_connection(); + + char data_[] = { 0, 0 }; + std::span data(reinterpret_cast(data_), sizeof(data_)); + for (auto& [key, sink] : metadata_sinks) { + CHECK(sink); + // we need to write some data to the sink to ensure it is created + CHECK(sink->write(0, data)); + CHECK(zarr::finalize_sink(std::move(sink))); // close the connection + + std::string path = test_dir + "/" + key; + CHECK(conn->object_exists(bucket_name, path)); + // cleanup + CHECK(conn->delete_object(bucket_name, path)); + } + + CHECK(conn->delete_object(bucket_name, "meta")); +} + +int +main() +{ + Logger::set_log_level(LogLevel_Debug); + + auto thread_pool = std::make_shared( + std::thread::hardware_concurrency(), + [](const std::string& err) { LOG_ERROR("Failed: ", err.c_str()); }); + + try { + sink_creator_make_v2_metadata_sinks(thread_pool); + sink_creator_make_v3_metadata_sinks(thread_pool); + } catch (const std::exception& e) { + LOG_ERROR("Failed: ", e.what()); + return 1; + } + + std::string s3_endpoint, bucket_name, s3_access_key_id, + s3_secret_access_key; + if (!get_credentials( + s3_endpoint, bucket_name, s3_access_key_id, s3_secret_access_key)) { + LOG_WARNING("Failed to get credentials. Skipping S3 portion of test."); + return 0; + } + + auto connection_pool = std::make_shared( + 4, s3_endpoint, s3_access_key_id, s3_secret_access_key); + + try { + sink_creator_make_v2_metadata_sinks( + thread_pool, connection_pool, bucket_name); + sink_creator_make_v3_metadata_sinks( + thread_pool, connection_pool, bucket_name); + } catch (const std::exception& e) { + LOG_ERROR("Failed: ", e.what()); + return 1; + } + + return 0; +} \ No newline at end of file diff --git a/tests/unit-tests/thread-pool-push-to-job-queue.cpp b/tests/unit-tests/thread-pool-push-to-job-queue.cpp index 12c26e08..72ef6ca8 100644 --- a/tests/unit-tests/thread-pool-push-to-job-queue.cpp +++ b/tests/unit-tests/thread-pool-push-to-job-queue.cpp @@ -1,7 +1,6 @@ #include "thread.pool.hh" #include "unit.test.macros.hh" -#include #include #include #include @@ -14,47 +13,42 @@ main() int retval = 0; fs::path tmp_path = fs::temp_directory_path() / TEST; - CHECK(!fs::exists(tmp_path)); - zarr::ThreadPool pool{ 1, [](const std::string&) {} }; + try { + CHECK(!fs::exists(tmp_path)); - CHECK(pool.push_job([&tmp_path](std::string&) { - std::ofstream ofs(tmp_path); - ofs << "Hello, Acquire!"; - ofs.close(); - return true; - })); - pool.await_stop(); + zarr::ThreadPool pool{ 1, [](const std::string&) {} }; - CHECK(fs::exists(tmp_path)); + CHECK(pool.push_job([&tmp_path](std::string&) { + std::ofstream ofs(tmp_path); + ofs << "Hello, Acquire!"; + ofs.close(); + return true; + })); + pool.await_stop(); - std::ifstream ifs(tmp_path); - CHECK(ifs.is_open()); + CHECK(fs::exists(tmp_path)); - std::string contents; - while (!ifs.eof()) { - std::getline(ifs, contents); - } - ifs.close(); + std::ifstream ifs(tmp_path); + CHECK(ifs.is_open()); + + std::string contents; + while (!ifs.eof()) { + std::getline(ifs, contents); + } + ifs.close(); - if (contents != "Hello, Acquire!") { - fprintf(stderr, - "Expected 'Hello, Acquire!' but got '%s'\n", - contents.c_str()); + EXPECT_STR_EQ(contents.c_str(), "Hello, Acquire!"); + } catch (const std::exception& e) { + LOG_ERROR("Caught exception: ", e.what()); retval = 1; } - goto Cleanup; - -Finalize: - return retval; - -Cleanup: std::error_code ec; if (!fs::remove(tmp_path, ec)) { - fprintf(stderr, "Failed to remove file: %s\n", ec.message().c_str()); + LOG_ERROR("Failed to remove file: ", ec.message()); retval = 1; } - goto Finalize; + return retval; } \ No newline at end of file diff --git a/tests/unit-tests/unit.test.macros.hh b/tests/unit-tests/unit.test.macros.hh index baacede4..302a033a 100644 --- a/tests/unit-tests/unit.test.macros.hh +++ b/tests/unit-tests/unit.test.macros.hh @@ -9,15 +9,16 @@ throw std::runtime_error(__err); \ } \ } while (0) -#define CHECK(e) EXPECT(e, "Expression evaluated as false:\n\t%s", #e) +#define CHECK(e) EXPECT(e, "Expression evaluated as false:\n\t", #e) /// Check that a==b -/// example: `ASSERT_EQ(int,"%d",42,meaning_of_life())` -#define EXPECT_EQ(T, fmt, a, b) \ +/// example: `ASSERT_EQ(int,42,meaning_of_life())` +#define EXPECT_EQ(T, a, b) \ do { \ T a_ = (T)(a); \ T b_ = (T)(b); \ - EXPECT(a_ == b_, "Expected %s==%s but " fmt "!=" fmt, #a, #b, a_, b_); \ + EXPECT( \ + a_ == b_, "Expected ", #a, " == ", #b, " but ", a_, " != ", b_); \ } while (0) #define EXPECT_STR_EQ(a, b) \ @@ -25,12 +26,16 @@ std::string a_ = (a) ? (a) : ""; \ std::string b_ = (b) ? (b) : ""; \ EXPECT(a_ == b_, \ - "Expected %s==%s but \"%s\"!=\"%s\"", \ + "Expected ", \ #a, \ + " == ", \ #b, \ - a_.c_str(), \ - b_.c_str()); \ + " but ", \ + a_, \ + " != ", \ + b_, \ + #a, \ + #b, \ + a_, \ + b_); \ } while (0) - -#define EXPECT_INT_EQ(a, b) \ - EXPECT((a) == (b), "Expected ", #a, " == ", #b, ", but ", a, " != ", b) \ No newline at end of file