From b0540fdc0d2313fa3e86b8c269ad58735fd150ba Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Fri, 4 Oct 2024 14:10:12 -0700 Subject: [PATCH] Standalone 14/14: Use the streaming API in the driver (#306) Depends on #305 --- acquire-common | 2 +- src/driver/CMakeLists.txt | 33 +- src/driver/README.md | 46 - src/driver/common/dimension.cpp | 79 -- src/driver/common/dimension.hh | 26 - src/driver/common/macros.hh | 20 - src/driver/common/s3.connection.cpp | 456 -------- src/driver/common/s3.connection.hh | 130 --- src/driver/common/thread.pool.cpp | 157 --- src/driver/common/thread.pool.hh | 50 - src/driver/common/utilities.cpp | 579 ----------- src/driver/common/utilities.hh | 124 --- src/driver/writers/array.writer.cpp | 975 ------------------ src/driver/writers/array.writer.hh | 88 -- src/driver/writers/blosc.compressor.cpp | 37 - src/driver/writers/blosc.compressor.hh | 53 - src/driver/writers/file.sink.cpp | 23 - src/driver/writers/file.sink.hh | 24 - src/driver/writers/s3.sink.cpp | 170 --- src/driver/writers/s3.sink.hh | 60 -- src/driver/writers/sink.creator.cpp | 544 ---------- src/driver/writers/sink.creator.hh | 109 -- src/driver/writers/sink.hh | 24 - src/driver/writers/zarrv2.array.writer.cpp | 554 ---------- src/driver/writers/zarrv2.array.writer.hh | 34 - src/driver/writers/zarrv3.array.writer.cpp | 733 ------------- src/driver/writers/zarrv3.array.writer.hh | 37 - src/driver/zarr.cpp | 937 ----------------- src/driver/zarr.hh | 104 -- src/driver/zarr.storage.cpp | 931 +++++++++++++++++ src/driver/zarr.storage.hh | 55 + src/driver/zarr.v2.cpp | 152 --- src/driver/zarr.v2.hh | 33 - src/driver/zarr.v3.cpp | 159 --- src/driver/zarr.v3.hh | 32 - src/streaming/CMakeLists.txt | 1 + src/streaming/zarr.stream.hh | 2 +- tests/driver/CMakeLists.txt | 1 - tests/driver/get-set-get.cpp | 24 +- tests/driver/get.cpp | 24 +- tests/driver/metadata-dimension-sizes.cpp | 32 +- tests/driver/multiscales-metadata.cpp | 21 +- tests/driver/repeat-start.cpp | 30 +- ...restart-stopped-zarr-resets-threadpool.cpp | 4 +- tests/driver/unit-tests.cpp | 120 --- .../write-zarr-v2-compressed-multiscale.cpp | 33 +- ...-compressed-with-chunking-and-rollover.cpp | 33 +- ...write-zarr-v2-compressed-with-chunking.cpp | 33 +- ...-raw-chunk-size-larger-than-frame-size.cpp | 33 +- ...-raw-multiscale-with-trivial-tile-size.cpp | 33 +- tests/driver/write-zarr-v2-raw-multiscale.cpp | 33 +- ...v2-raw-with-even-chunking-and-rollover.cpp | 33 +- .../write-zarr-v2-raw-with-even-chunking.cpp | 33 +- ...write-zarr-v2-raw-with-ragged-chunking.cpp | 33 +- tests/driver/write-zarr-v2-raw.cpp | 33 +- tests/driver/write-zarr-v2-to-s3.cpp | 39 +- .../write-zarr-v2-with-lz4-compression.cpp | 33 +- .../write-zarr-v2-with-zstd-compression.cpp | 33 +- tests/driver/write-zarr-v3-compressed.cpp | 18 +- .../write-zarr-v3-raw-chunk-exceeds-array.cpp | 26 +- tests/driver/write-zarr-v3-raw-multiscale.cpp | 30 +- ...write-zarr-v3-raw-with-ragged-sharding.cpp | 56 +- tests/driver/write-zarr-v3-raw.cpp | 40 +- tests/driver/write-zarr-v3-to-s3.cpp | 31 +- 64 files changed, 1379 insertions(+), 7086 deletions(-) delete mode 100644 src/driver/README.md delete mode 100644 src/driver/common/dimension.cpp delete mode 100644 src/driver/common/dimension.hh delete mode 100644 src/driver/common/macros.hh delete mode 100644 src/driver/common/s3.connection.cpp delete mode 100644 src/driver/common/s3.connection.hh delete mode 100644 src/driver/common/thread.pool.cpp delete mode 100644 src/driver/common/thread.pool.hh delete mode 100644 src/driver/common/utilities.cpp delete mode 100644 src/driver/common/utilities.hh delete mode 100644 src/driver/writers/array.writer.cpp delete mode 100644 src/driver/writers/array.writer.hh delete mode 100644 src/driver/writers/blosc.compressor.cpp delete mode 100644 src/driver/writers/blosc.compressor.hh delete mode 100644 src/driver/writers/file.sink.cpp delete mode 100644 src/driver/writers/file.sink.hh delete mode 100644 src/driver/writers/s3.sink.cpp delete mode 100644 src/driver/writers/s3.sink.hh delete mode 100644 src/driver/writers/sink.creator.cpp delete mode 100644 src/driver/writers/sink.creator.hh delete mode 100644 src/driver/writers/sink.hh delete mode 100644 src/driver/writers/zarrv2.array.writer.cpp delete mode 100644 src/driver/writers/zarrv2.array.writer.hh delete mode 100644 src/driver/writers/zarrv3.array.writer.cpp delete mode 100644 src/driver/writers/zarrv3.array.writer.hh delete mode 100644 src/driver/zarr.cpp delete mode 100644 src/driver/zarr.hh create mode 100644 src/driver/zarr.storage.cpp create mode 100644 src/driver/zarr.storage.hh delete mode 100644 src/driver/zarr.v2.cpp delete mode 100644 src/driver/zarr.v2.hh delete mode 100644 src/driver/zarr.v3.cpp delete mode 100644 src/driver/zarr.v3.hh delete mode 100644 tests/driver/unit-tests.cpp diff --git a/acquire-common b/acquire-common index bbd03f38..8e3e9957 160000 --- a/acquire-common +++ b/acquire-common @@ -1 +1 @@ -Subproject commit bbd03f38d2ce48d25767defd3da49e4297d4be1c +Subproject commit 8e3e9957c6f5d66753dc0a312c3a729e98f8e737 diff --git a/src/driver/CMakeLists.txt b/src/driver/CMakeLists.txt index 5b72798a..2ca6a2f2 100644 --- a/src/driver/CMakeLists.txt +++ b/src/driver/CMakeLists.txt @@ -4,35 +4,8 @@ endif () set(tgt acquire-driver-zarr) add_library(${tgt} MODULE - common/dimension.hh - common/dimension.cpp - common/thread.pool.hh - common/thread.pool.cpp - common/s3.connection.hh - common/s3.connection.cpp - common/utilities.hh - common/utilities.cpp - writers/sink.hh - writers/sink.creator.hh - writers/sink.creator.cpp - writers/file.sink.hh - writers/file.sink.cpp - writers/s3.sink.hh - writers/s3.sink.cpp - writers/array.writer.hh - writers/array.writer.cpp - writers/zarrv2.array.writer.hh - writers/zarrv2.array.writer.cpp - writers/zarrv3.array.writer.hh - writers/zarrv3.array.writer.cpp - writers/blosc.compressor.hh - writers/blosc.compressor.cpp - zarr.hh - zarr.cpp - zarr.v2.hh - zarr.v2.cpp - zarr.v3.hh - zarr.v3.cpp + zarr.storage.hh + zarr.storage.cpp zarr.driver.c ) @@ -46,10 +19,12 @@ target_link_libraries(${tgt} PRIVATE acquire-core-platform acquire-device-kit acquire-device-properties + acquire-zarr blosc_static nlohmann_json::nlohmann_json miniocpp::miniocpp ) + set_target_properties(${tgt} PROPERTIES MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" ) diff --git a/src/driver/README.md b/src/driver/README.md deleted file mode 100644 index ba076a5d..00000000 --- a/src/driver/README.md +++ /dev/null @@ -1,46 +0,0 @@ -# The Zarr Storage device - -## Components - -### The `Zarr` class - -An abstract class that implements the `Storage` device interface. -Zarr -is "[a file storage format for chunked, compressed, N-dimensional arrays based on an open-source specification.](https://zarr.readthedocs.io/en/stable/index.html)" - -### The `ZarrV2` class - -Subclass of the `Zarr` class. -Implements abstract methods for writer allocation and metadata. -Specifically, `ZarrV2` allocates one writer of type `ZarrV2ArrayWriter` for each multiscale level-of-detail -and writes metadata in the format specified by the [Zarr V2 spec](https://zarr.readthedocs.io/en/stable/spec/v2.html). - -### The `ZarrV3` class - -Subclass of the `Zarr` class. -Implements abstract methods for writer allocation and metadata. -Specifically, `ZarrV3` allocates one writer of type `ZarrV3ArrayWriter` for each multiscale level-of-detail -and writes metadata in the format specified by -the [Zarr V3 spec](https://zarr-specs.readthedocs.io/en/latest/specs.html). - -### The `ArrayWriter` class - -An abstract class that writes frames to the filesystem or other storage layer. -In general, frames are chunked and potentially compressed. -The `ArrayWriter` handles chunking, chunk compression, and writing. - -### The `ZarrV2ArrayWriter` class - -Subclass of the `ArrayWriter` class. -Implements abstract methods relating to writing and flushing chunk buffers. -Chunk buffers, whether raw or compressed, are written to individual chunk files. - -### The `ZarrV3ArrayWriter` class - -Subclass of the `ArrayWriter` class. -Implements abstract methods relating to writing, sharding, and flushing chunk buffers. -Chunk buffers, whether raw or compressed, are concatenated into shards, which are written out to individual shard files. - -### The `BloscCompressionParams` struct - -Stores parameters for compression using [C-Blosc](https://github.com/Blosc/c-blosc). diff --git a/src/driver/common/dimension.cpp b/src/driver/common/dimension.cpp deleted file mode 100644 index e724acdd..00000000 --- a/src/driver/common/dimension.cpp +++ /dev/null @@ -1,79 +0,0 @@ -#include "dimension.hh" - -#include - -namespace zarr = acquire::sink::zarr; - -namespace { -inline std::string -trim(const std::string& s) -{ - // trim left - std::string trimmed = s; - trimmed.erase(trimmed.begin(), - std::find_if(trimmed.begin(), trimmed.end(), [](char c) { - return !std::isspace(c); - })); - - // trim right - trimmed.erase(std::find_if(trimmed.rbegin(), - trimmed.rend(), - [](char c) { return !std::isspace(c); }) - .base(), - trimmed.end()); - - return trimmed; -} -} // namespace - -zarr::Dimension::Dimension(const std::string& name, - DimensionType kind, - uint32_t array_size_px, - uint32_t chunk_size_px, - uint32_t shard_size_chunks) - : name{ trim(name) } - , kind{ kind } - , array_size_px{ array_size_px } - , chunk_size_px{ chunk_size_px } - , shard_size_chunks{ shard_size_chunks } -{ - EXPECT(kind < DimensionTypeCount, "Invalid dimension type."); - EXPECT(!name.empty(), "Dimension name cannot be empty."); -} - -zarr::Dimension::Dimension(const StorageDimension& dim) - : Dimension(dim.name.str, - dim.kind, - dim.array_size_px, - dim.chunk_size_px, - dim.shard_size_chunks) -{ -} - -#ifndef NO_UNIT_TESTS -#ifdef _WIN32 -#define acquire_export __declspec(dllexport) -#else -#define acquire_export -#endif // _WIN32 - -extern "C" acquire_export int -unit_test__trim() -{ - try { - EXPECT(trim(" foo") == "foo", "Failed to trim left."); - EXPECT(trim("foo ") == "foo", "Failed to trim right."); - EXPECT(trim(" foo ") == "foo", "Failed to trim both."); - EXPECT(trim("foo") == "foo", "Failed to trim either."); - EXPECT(trim("").empty(), "Failed to trim empty."); - return 1; - } catch (const std::exception& e) { - LOGE("Exception: %s", e.what()); - } catch (...) { - LOGE("Unknown exception"); - } - - return 0; -} - -#endif // NO_UNIT_TESTS diff --git a/src/driver/common/dimension.hh b/src/driver/common/dimension.hh deleted file mode 100644 index 7cf14188..00000000 --- a/src/driver/common/dimension.hh +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once - -#include "macros.hh" -#include "device/props/storage.h" - -#include - -namespace acquire::sink::zarr { -struct Dimension -{ - public: - explicit Dimension(const std::string& name, - DimensionType kind, - unsigned int array_size_px, - unsigned int chunk_size_px, - unsigned int shard_size_chunks); - - explicit Dimension(const StorageDimension& dim); - - const std::string name; - const DimensionType kind; - const unsigned int array_size_px; - const unsigned int chunk_size_px; - const unsigned int shard_size_chunks; -}; -} // namespace acquire::sink::zarr diff --git a/src/driver/common/macros.hh b/src/driver/common/macros.hh deleted file mode 100644 index b4a5c263..00000000 --- a/src/driver/common/macros.hh +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include "logger.h" -#include - -#define LOG(...) aq_logger(0, __FILE__, __LINE__, __FUNCTION__, __VA_ARGS__) -#define LOGE(...) aq_logger(1, __FILE__, __LINE__, __FUNCTION__, __VA_ARGS__) -#define EXPECT(e, ...) \ - do { \ - if (!(e)) { \ - LOGE(__VA_ARGS__); \ - throw std::runtime_error("Expression was false: " #e); \ - } \ - } while (0) -#define CHECK(e) EXPECT(e, "Expression evaluated as false:\n\t%s", #e) - -#define TRACE(...) - -#define containerof(ptr, T, V) ((T*)(((char*)(ptr)) - offsetof(T, V))) -#define countof(e) (sizeof(e) / sizeof(*(e))) \ No newline at end of file diff --git a/src/driver/common/s3.connection.cpp b/src/driver/common/s3.connection.cpp deleted file mode 100644 index bb848457..00000000 --- a/src/driver/common/s3.connection.cpp +++ /dev/null @@ -1,456 +0,0 @@ -#include "s3.connection.hh" -#include "utilities.hh" - -#include - -#include -#include -#include - -namespace zarr = acquire::sink::zarr; -namespace common = zarr::common; - -common::S3Connection::S3Connection(const std::string& endpoint, - const std::string& access_key_id, - const std::string& secret_access_key) -{ - minio::s3::BaseUrl url(endpoint); - url.https = endpoint.starts_with("https"); - - provider_ = std::make_unique( - access_key_id, secret_access_key); - client_ = std::make_unique(url, provider_.get()); - CHECK(client_); -} - -bool -common::S3Connection::check_connection() -{ - return static_cast(client_->ListBuckets()); -} - -bool -common::S3Connection::bucket_exists(std::string_view bucket_name) -{ - EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); - - minio::s3::BucketExistsArgs args; - args.bucket = bucket_name; - - auto response = client_->BucketExists(args); - return response.exist; -} - -bool -common::S3Connection::object_exists(std::string_view bucket_name, - std::string_view object_name) -{ - EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); - EXPECT(!object_name.empty(), "Object name must not be empty."); - - minio::s3::StatObjectArgs args; - args.bucket = bucket_name; - args.object = object_name; - - auto response = client_->StatObject(args); - // casts to true if response code in 200 range and error message is empty - return static_cast(response); -} - -std::string -common::S3Connection::put_object(std::string_view bucket_name, - std::string_view object_name, - std::span data) -{ - EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); - EXPECT(!object_name.empty(), "Object name must not be empty."); - EXPECT(!data.empty(), "Data must not be empty."); - - minio::utils::CharBuffer buffer((char*)const_cast(data.data()), - data.size()); - std::basic_istream stream(&buffer); - - TRACE( - "Putting object %s in bucket %s", object_name.data(), bucket_name.data()); - minio::s3::PutObjectArgs args(stream, (long)data.size(), 0); - args.bucket = bucket_name; - args.object = object_name; - - auto response = client_->PutObject(args); - if (!response) { - LOGE("Failed to put object %s in bucket %s: %s", - object_name.data(), - bucket_name.data(), - response.Error().String().c_str()); - return {}; - } - - return response.etag; -} - -bool -common::S3Connection::delete_object(std::string_view bucket_name, - std::string_view object_name) -{ - EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); - EXPECT(!object_name.empty(), "Object name must not be empty."); - - TRACE("Deleting object %s from bucket %s", - object_name.data(), - bucket_name.data()); - minio::s3::RemoveObjectArgs args; - args.bucket = bucket_name; - args.object = object_name; - - auto response = client_->RemoveObject(args); - if (!response) { - LOGE("Failed to delete object %s from bucket %s: %s", - object_name.data(), - bucket_name.data(), - response.Error().String().c_str()); - return false; - } - - return true; -} - -std::string -common::S3Connection::create_multipart_object(std::string_view bucket_name, - std::string_view object_name) -{ - EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); - EXPECT(!object_name.empty(), "Object name must not be empty."); - - TRACE("Creating multipart object %s in bucket %s", - object_name.data(), - bucket_name.data()); - minio::s3::CreateMultipartUploadArgs args; - args.bucket = bucket_name; - args.object = object_name; - - auto response = client_->CreateMultipartUpload(args); - CHECK(!response.upload_id.empty()); - - return response.upload_id; -} - -std::string -common::S3Connection::upload_multipart_object_part(std::string_view bucket_name, - std::string_view object_name, - std::string_view upload_id, - std::span data, - unsigned int part_number) -{ - EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); - EXPECT(!object_name.empty(), "Object name must not be empty."); - EXPECT(!data.empty(), "Number of bytes must be positive."); - EXPECT(part_number, "Part number must be positive."); - - TRACE("Uploading multipart object part %zu for object %s in bucket %s", - part_number, - object_name.data(), - bucket_name.data()); - - std::string_view sv(reinterpret_cast(data.data()), - data.size()); - - minio::s3::UploadPartArgs args; - args.bucket = bucket_name; - args.object = object_name; - args.part_number = part_number; - args.upload_id = upload_id; - args.data = sv; - - auto response = client_->UploadPart(args); - if (!response) { - LOGE("Failed to upload part %zu for object %s in bucket %s: %s", - part_number, - object_name.data(), - bucket_name.data(), - response.Error().String().c_str()); - return {}; - } - - return response.etag; -} - -bool -common::S3Connection::complete_multipart_object( - std::string_view bucket_name, - std::string_view object_name, - std::string_view upload_id, - const std::list& parts) -{ - EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); - EXPECT(!object_name.empty(), "Object name must not be empty."); - EXPECT(!upload_id.empty(), "Upload id must not be empty."); - EXPECT(!parts.empty(), "Parts list must not be empty."); - - TRACE("Completing multipart object %s in bucket %s", - object_name.data(), - bucket_name.data()); - minio::s3::CompleteMultipartUploadArgs args; - args.bucket = bucket_name; - args.object = object_name; - args.upload_id = upload_id; - args.parts = parts; - - auto response = client_->CompleteMultipartUpload(args); - if (!response) { - LOGE("Failed to complete multipart object %s in bucket %s: %s", - object_name.data(), - bucket_name.data(), - response.Error().String().c_str()); - return false; - } - - return true; -} - -common::S3ConnectionPool::S3ConnectionPool(size_t n_connections, - const std::string& endpoint, - const std::string& access_key_id, - const std::string& secret_access_key) - : is_accepting_connections_{ true } -{ - for (auto i = 0; i < n_connections; ++i) { - auto connection = std::make_unique( - endpoint, access_key_id, secret_access_key); - - if (connection->check_connection()) { - connections_.push_back(std::make_unique( - endpoint, access_key_id, secret_access_key)); - } - } - - CHECK(!connections_.empty()); -} - -common::S3ConnectionPool::~S3ConnectionPool() noexcept -{ - is_accepting_connections_ = false; - cv_.notify_all(); -} - -std::unique_ptr -common::S3ConnectionPool::get_connection() -{ - std::unique_lock lock(connections_mutex_); - cv_.wait(lock, [this] { return !connections_.empty(); }); - - if (!is_accepting_connections_ || connections_.empty()) { - return nullptr; - } - - auto conn = std::move(connections_.back()); - connections_.pop_back(); - return conn; -} - -void -common::S3ConnectionPool::return_connection( - std::unique_ptr&& conn) -{ - std::scoped_lock lock(connections_mutex_); - connections_.push_back(std::move(conn)); - cv_.notify_one(); -} - -#ifndef NO_UNIT_TESTS -#ifdef _WIN32 -#define acquire_export __declspec(dllexport) -#else -#define acquire_export -#endif - -#include - -namespace { -bool -get_credentials(std::string& endpoint, - std::string& bucket_name, - std::string& access_key_id, - std::string& secret_access_key) -{ - char* env = nullptr; - if (!(env = std::getenv("ZARR_S3_ENDPOINT"))) { - LOGE("ZARR_S3_ENDPOINT not set."); - return false; - } - endpoint = env; - - if (!(env = std::getenv("ZARR_S3_BUCKET_NAME"))) { - LOGE("ZARR_S3_BUCKET_NAME not set."); - return false; - } - bucket_name = env; - - if (!(env = std::getenv("ZARR_S3_ACCESS_KEY_ID"))) { - LOGE("ZARR_S3_ACCESS_KEY_ID not set."); - return false; - } - access_key_id = env; - - if (!(env = std::getenv("ZARR_S3_SECRET_ACCESS_KEY"))) { - LOGE("ZARR_S3_SECRET_ACCESS_KEY not set."); - return false; - } - secret_access_key = env; - - return true; -} -} // namespace - -extern "C" -{ - acquire_export int unit_test__s3_connection__put_object() - { - std::string s3_endpoint, bucket_name, s3_access_key_id, - s3_secret_access_key; - if (!get_credentials(s3_endpoint, - bucket_name, - s3_access_key_id, - s3_secret_access_key)) { - return 1; - } - - minio::s3::BaseUrl url(s3_endpoint); - url.https = s3_endpoint.starts_with("https"); - - minio::creds::StaticProvider provider(s3_access_key_id, - s3_secret_access_key); - minio::s3::Client client(url, &provider); - - const std::string object_name = "test-object"; - - int retval = 0; - - try { - common::S3Connection conn( - s3_endpoint, s3_access_key_id, s3_secret_access_key); - - CHECK(conn.bucket_exists(bucket_name)); - - CHECK(conn.delete_object(bucket_name, object_name)); - CHECK(!conn.object_exists(bucket_name, object_name)); - - std::vector data(1024, 0); - - std::string etag = - conn.put_object(bucket_name, - object_name, - std::span(data.data(), data.size())); - CHECK(!etag.empty()); - - CHECK(conn.object_exists(bucket_name, object_name)); - - // cleanup - CHECK(conn.delete_object(bucket_name, object_name)); - - retval = 1; - } catch (const std::exception& e) { - LOGE("Failed to create S3 connection: %s", e.what()); - } catch (...) { - LOGE("Failed to create S3 connection: unknown error"); - } - - return retval; - } - - acquire_export int unit_test__s3_connection__upload_multipart_object() - { - std::string s3_endpoint, bucket_name, s3_access_key_id, - s3_secret_access_key; - if (!get_credentials(s3_endpoint, - bucket_name, - s3_access_key_id, - s3_secret_access_key)) { - return 1; - } - - minio::s3::BaseUrl url(s3_endpoint); - url.https = s3_endpoint.starts_with("https"); - - minio::creds::StaticProvider provider(s3_access_key_id, - s3_secret_access_key); - minio::s3::Client client(url, &provider); - - const std::string object_name = "test-object"; - - int retval = 0; - - try { - common::S3Connection conn( - s3_endpoint, s3_access_key_id, s3_secret_access_key); - - CHECK(conn.bucket_exists(bucket_name)); - - if (conn.object_exists(bucket_name, object_name)) { - CHECK(conn.delete_object(bucket_name, object_name)); - CHECK(!conn.object_exists(bucket_name, object_name)); - } - - std::string upload_id = - conn.create_multipart_object(bucket_name, object_name); - CHECK(!upload_id.empty()); - - std::list parts; - - // parts need to be at least 5MiB, except the last part - std::vector data(5 << 20, 0); - for (auto i = 0; i < 4; ++i) { - std::string etag = conn.upload_multipart_object_part( - bucket_name, - object_name, - upload_id, - std::span(data.data(), data.size()), - i + 1); - CHECK(!etag.empty()); - - minio::s3::Part part; - part.number = i + 1; - part.etag = etag; - part.size = data.size(); - - parts.push_back(part); - } - - // last part is 1MiB - { - const unsigned int part_number = parts.size() + 1; - const size_t part_size = 1 << 20; // 1MiB - std::string etag = conn.upload_multipart_object_part( - bucket_name, - object_name, - upload_id, - std::span(data.data(), data.size()), - part_number); - CHECK(!etag.empty()); - - minio::s3::Part part; - part.number = part_number; - part.etag = etag; - part.size = part_size; - - parts.push_back(part); - } - - CHECK(conn.complete_multipart_object( - bucket_name, object_name, upload_id, parts)); - - CHECK(conn.object_exists(bucket_name, object_name)); - - // cleanup - CHECK(conn.delete_object(bucket_name, object_name)); - - retval = 1; - } catch (const std::exception& e) { - LOGE("Failed to create S3 connection: %s", e.what()); - } catch (...) { - LOGE("Failed to create S3 connection: unknown error"); - } - - return retval; - } -} -#endif diff --git a/src/driver/common/s3.connection.hh b/src/driver/common/s3.connection.hh deleted file mode 100644 index b966e1f4..00000000 --- a/src/driver/common/s3.connection.hh +++ /dev/null @@ -1,130 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace acquire::sink::zarr::common { -struct S3Connection final -{ - S3Connection() = delete; - explicit S3Connection(const std::string& endpoint, - const std::string& access_key_id, - const std::string& secret_access_key); - - ~S3Connection() noexcept = default; - - /// @brief Test a connection by listing all buckets at this connection's - /// endpoint. - /// @returns True if the connection is valid, otherwise false. - bool check_connection(); - - // bucket operations - /// @brief Check whether a bucket exists. - /// @param bucket_name The name of the bucket. - /// @returns True if the bucket exists, otherwise false. - /// @throws std::runtime_error if the bucket name is empty. - bool bucket_exists(std::string_view bucket_name); - - // object operations - /// @brief Check whether an object exists. - /// @param bucket_name The name of the bucket containing the object. - /// @param object_name The name of the object. - /// @returns True if the object exists, otherwise false. - /// @throws std::runtime_error if the bucket name is empty or the object - /// name is empty. - bool object_exists(std::string_view bucket_name, - std::string_view object_name); - - /// @brief Put an object. - /// @param bucket_name The name of the bucket to put the object in. - /// @param object_name The name of the object. - /// @param data The data to put in the object. - /// @returns The etag of the object. - /// @throws std::runtime_error if the bucket name is empty, the object name - /// is empty, or @p data is empty. - [[nodiscard]] std::string put_object(std::string_view bucket_name, - std::string_view object_name, - std::span data); - - /// @brief Delete an object. - /// @param bucket_name The name of the bucket containing the object. - /// @param object_name The name of the object. - /// @returns True if the object was successfully deleted, otherwise false. - /// @throws std::runtime_error if the bucket name is empty or the object - /// name is empty. - [[nodiscard]] bool delete_object(std::string_view bucket_name, - std::string_view object_name); - - // multipart object operations - /// @brief Create a multipart object. - /// @param bucket_name The name of the bucket containing the object. - /// @param object_name The name of the object. - /// @returns The upload id of the multipart object. Nonempty if and only if - /// the operation succeeds. - /// @throws std::runtime_error if the bucket name is empty or the object - /// name is empty. - [[nodiscard]] std::string create_multipart_object( - std::string_view bucket_name, - std::string_view object_name); - - /// @brief Upload a part of a multipart object. - /// @param bucket_name The name of the bucket containing the object. - /// @param object_name The name of the object. - /// @param upload_id The upload id of the multipart object. - /// @param data The data to upload. - /// @param part_number The part number of the object. - /// @returns The etag of the uploaded part. Nonempty if and only if the - /// operation is successful. - /// @throws std::runtime_error if the bucket name is empty, the object name - /// is empty, @p data is empty, or @p part_number is 0. - [[nodiscard]] std::string upload_multipart_object_part( - std::string_view bucket_name, - std::string_view object_name, - std::string_view upload_id, - std::span data, - unsigned int part_number); - - /// @brief Complete a multipart object. - /// @param bucket_name The name of the bucket containing the object. - /// @param object_name The name of the object. - /// @param upload_id The upload id of the multipart object. - /// @param parts List of the parts making up the object. - /// @returns True if the object was successfully completed, otherwise false. - [[nodiscard]] bool complete_multipart_object( - std::string_view bucket_name, - std::string_view object_name, - std::string_view upload_id, - const std::list& parts); - - private: - std::unique_ptr client_; - std::unique_ptr provider_; -}; - -struct S3ConnectionPool final -{ - public: - S3ConnectionPool(size_t n_connections, - const std::string& endpoint, - const std::string& access_key_id, - const std::string& secret_access_key); - ~S3ConnectionPool() noexcept; - - std::unique_ptr get_connection(); - void return_connection(std::unique_ptr&& conn); - - private: - std::vector> connections_; - mutable std::mutex connections_mutex_; - std::condition_variable cv_; - - std::atomic is_accepting_connections_; -}; -} // namespace acquire::sink::zarr::common diff --git a/src/driver/common/thread.pool.cpp b/src/driver/common/thread.pool.cpp deleted file mode 100644 index 0071fbff..00000000 --- a/src/driver/common/thread.pool.cpp +++ /dev/null @@ -1,157 +0,0 @@ -#include "thread.pool.hh" -#include "utilities.hh" - -namespace zarr = acquire::sink::zarr; -namespace common = zarr::common; - -common::ThreadPool::ThreadPool(unsigned int n_threads, - std::function err) - : error_handler_{ err } - , is_accepting_jobs_{ true } -{ - n_threads = std::clamp( - n_threads, 1u, std::max(std::thread::hardware_concurrency(), 1u)); - - for (auto i = 0; i < n_threads; ++i) { - threads_.emplace_back([this] { thread_worker_(); }); - } -} - -common::ThreadPool::~ThreadPool() noexcept -{ - { - std::scoped_lock lock(jobs_mutex_); - while (!jobs_.empty()) { - jobs_.pop(); - } - } - - await_stop(); -} - -void -common::ThreadPool::push_to_job_queue(JobT&& job) -{ - std::unique_lock lock(jobs_mutex_); - CHECK(is_accepting_jobs_); - jobs_.push(std::move(job)); - - cv_.notify_one(); -} - -void -common::ThreadPool::await_stop() noexcept -{ - { - std::scoped_lock lock(jobs_mutex_); - is_accepting_jobs_ = false; - - cv_.notify_all(); - } - - // spin down threads - for (auto& thread : threads_) { - if (thread.joinable()) { - thread.join(); - } - } -} - -std::optional -common::ThreadPool::pop_from_job_queue_() noexcept -{ - if (jobs_.empty()) { - return std::nullopt; - } - - auto job = std::move(jobs_.front()); - jobs_.pop(); - return job; -} - -bool -common::ThreadPool::should_stop_() const noexcept -{ - return !is_accepting_jobs_ && jobs_.empty(); -} - -void -common::ThreadPool::thread_worker_() -{ - TRACE("Worker thread starting."); - - while (true) { - std::unique_lock lock(jobs_mutex_); - cv_.wait(lock, [&] { return should_stop_() || !jobs_.empty(); }); - - if (should_stop_()) { - break; - } - - if (auto job = pop_from_job_queue_(); job.has_value()) { - lock.unlock(); - if (std::string err_msg; !job.value()(err_msg)) { - error_handler_(err_msg); - } - } - } - - TRACE("Worker thread exiting."); -} - -#ifndef NO_UNIT_TESTS -#ifdef _WIN32 -#define acquire_export __declspec(dllexport) -#else -#define acquire_export -#endif // _WIN32 - -#include -#include -#include - -namespace fs = std::filesystem; - -extern "C" -{ - acquire_export int unit_test__thread_pool__push_to_job_queue() - { - int retval = 0; - - fs::path tmp_path = fs::temp_directory_path() / "tmp_file"; - - try { - CHECK(!fs::exists(tmp_path)); - - common::ThreadPool pool{ 1, [](const std::string&) {} }; - pool.push_to_job_queue([&tmp_path](std::string&) { - std::ofstream ofs(tmp_path); - ofs << "Hello, Acquire!"; - ofs.close(); - return true; - }); - pool.await_stop(); - - CHECK(fs::exists(tmp_path)); - - retval = 1; - } catch (const std::exception& exc) { - LOGE("Caught exception: %s", exc.what()); - } catch (...) { - LOGE("Caught unknown exception"); - } - - try { - // cleanup - if (fs::exists(tmp_path)) { - fs::remove(tmp_path); - } - } catch (const std::exception& exc) { - LOGE("Caught exception: %s", exc.what()); - retval = 0; - } - - return retval; - } -} -#endif // NO_UNIT_TESTS \ No newline at end of file diff --git a/src/driver/common/thread.pool.hh b/src/driver/common/thread.pool.hh deleted file mode 100644 index a5e24c01..00000000 --- a/src/driver/common/thread.pool.hh +++ /dev/null @@ -1,50 +0,0 @@ -#ifndef H_ACQUIRE_STORAGE_ZARR_THREAD_POOL_V0 -#define H_ACQUIRE_STORAGE_ZARR_THREAD_POOL_V0 - -#include -#include -#include -#include -#include -#include -#include - -namespace acquire::sink::zarr::common { -struct ThreadPool final -{ - public: - using JobT = std::function; - - // The error handler `err` is called when a job returns false. This - // can happen when the job encounters an error, or otherwise fails. The - // std::string& argument to the error handler is a diagnostic message from - // the failing job and is logged to the error stream by the Zarr driver when - // the next call to `append()` is made. - ThreadPool(unsigned int n_threads, std::function err); - ~ThreadPool() noexcept; - - void push_to_job_queue(JobT&& job); - - /** - * @brief Block until all jobs on the queue have processed, then spin down - * the threads. - * @note After calling this function, the job queue no longer accepts jobs. - */ - void await_stop() noexcept; - - private: - std::function error_handler_; - - std::vector threads_; - mutable std::mutex jobs_mutex_; - std::condition_variable cv_; - std::queue jobs_; - - std::atomic is_accepting_jobs_; - - std::optional pop_from_job_queue_() noexcept; - [[nodiscard]] bool should_stop_() const noexcept; - void thread_worker_(); -}; -} -#endif // H_ACQUIRE_STORAGE_ZARR_THREAD_POOL_V0 diff --git a/src/driver/common/utilities.cpp b/src/driver/common/utilities.cpp deleted file mode 100644 index 769468e6..00000000 --- a/src/driver/common/utilities.cpp +++ /dev/null @@ -1,579 +0,0 @@ -#include "utilities.hh" -#include "zarr.hh" - -#include "platform.h" -#include "thread.pool.hh" - -#include -#include - -namespace zarr = acquire::sink::zarr; -namespace common = zarr::common; - -size_t -common::chunks_along_dimension(const Dimension& dimension) -{ - EXPECT(dimension.chunk_size_px > 0, "Invalid chunk_size size."); - - return (dimension.array_size_px + dimension.chunk_size_px - 1) / - dimension.chunk_size_px; -} - -size_t -common::shards_along_dimension(const Dimension& dimension) -{ - const size_t shard_size = dimension.shard_size_chunks; - if (shard_size == 0) { - return 0; - } - - const size_t n_chunks = chunks_along_dimension(dimension); - return (n_chunks + shard_size - 1) / shard_size; -} - -size_t -common::number_of_chunks_in_memory(const std::vector& dimensions) -{ - size_t n_chunks = 1; - for (auto i = 0; i < dimensions.size() - 1; ++i) { - n_chunks *= chunks_along_dimension(dimensions[i]); - } - - return n_chunks; -} - -size_t -common::number_of_shards(const std::vector& dimensions) -{ - size_t n_shards = 1; - for (auto i = 0; i < dimensions.size() - 1; ++i) { - const auto& dim = dimensions.at(i); - n_shards *= shards_along_dimension(dim); - } - - return n_shards; -} - -size_t -common::chunks_per_shard(const std::vector& dimensions) -{ - size_t n_chunks = 1; - for (const auto& dim : dimensions) { - n_chunks *= dim.shard_size_chunks; - } - - return n_chunks; -} - -size_t -common::shard_index_for_chunk(size_t chunk_index, - const std::vector& dimensions) -{ - // make chunk strides - std::vector chunk_strides(1, 1); - for (auto i = 0; i < dimensions.size() - 1; ++i) { - const auto& dim = dimensions.at(i); - chunk_strides.push_back(chunk_strides.back() * - zarr::common::chunks_along_dimension(dim)); - CHECK(chunk_strides.back()); - } - - // get chunk indices - std::vector chunk_lattice_indices; - for (auto i = 0; i < dimensions.size() - 1; ++i) { - chunk_lattice_indices.push_back(chunk_index % chunk_strides.at(i + 1) / - chunk_strides.at(i)); - } - chunk_lattice_indices.push_back(chunk_index / chunk_strides.back()); - - // make shard strides - std::vector shard_strides(1, 1); - for (auto i = 0; i < dimensions.size() - 1; ++i) { - const auto& dim = dimensions.at(i); - shard_strides.push_back(shard_strides.back() * - zarr::common::shards_along_dimension(dim)); - } - - std::vector shard_lattice_indices; - for (auto i = 0; i < dimensions.size(); ++i) { - shard_lattice_indices.push_back(chunk_lattice_indices.at(i) / - dimensions.at(i).shard_size_chunks); - } - - size_t index = 0; - for (auto i = 0; i < dimensions.size(); ++i) { - index += shard_lattice_indices.at(i) * shard_strides.at(i); - } - - return index; -} - -size_t -common::shard_internal_index(size_t chunk_idx, - const std::vector& dimensions) -{ - // make chunk strides - std::vector chunk_strides(1, 1); - for (auto i = 0; i < dimensions.size() - 1; ++i) { - const auto& dim = dimensions.at(i); - chunk_strides.push_back(chunk_strides.back() * - zarr::common::chunks_along_dimension(dim)); - CHECK(chunk_strides.back()); - } - - // get chunk indices - std::vector chunk_lattice_indices; - for (auto i = 0; i < dimensions.size() - 1; ++i) { - chunk_lattice_indices.push_back(chunk_idx % chunk_strides.at(i + 1) / - chunk_strides.at(i)); - } - chunk_lattice_indices.push_back(chunk_idx / chunk_strides.back()); - - // make shard lattice indices - std::vector shard_lattice_indices; - for (auto i = 0; i < dimensions.size(); ++i) { - shard_lattice_indices.push_back(chunk_lattice_indices.at(i) / - dimensions.at(i).shard_size_chunks); - } - - std::vector chunk_internal_strides(1, 1); - for (auto i = 0; i < dimensions.size() - 1; ++i) { - const auto& dim = dimensions.at(i); - chunk_internal_strides.push_back(chunk_internal_strides.back() * - dim.shard_size_chunks); - } - - size_t index = 0; - - for (auto i = 0; i < dimensions.size(); ++i) { - index += - (chunk_lattice_indices.at(i) % dimensions.at(i).shard_size_chunks) * - chunk_internal_strides.at(i); - } - - return index; -} - -size_t -common::bytes_per_chunk(const std::vector& dimensions, - const SampleType& type) -{ - auto n_bytes = bytes_of_type(type); - for (const auto& d : dimensions) { - n_bytes *= d.chunk_size_px; - } - - return n_bytes; -} - -const char* -common::sample_type_to_string(SampleType t) noexcept -{ - switch (t) { - case SampleType_u8: - return "u8"; - case SampleType_u16: - return "u16"; - case SampleType_i8: - return "i8"; - case SampleType_i16: - return "i16"; - case SampleType_f32: - return "f32"; - default: - return "unrecognized pixel type"; - } -} - -size_t -common::align_up(size_t n, size_t align) -{ - EXPECT(align > 0, "Alignment must be greater than zero."); - return align * ((n + align - 1) / align); -} - -std::vector -common::split_uri(const std::string& uri) -{ - const char delim = '/'; - - std::vector out; - size_t begin = 0, end = uri.find_first_of(delim); - - while (end != std::string::npos) { - std::string part = uri.substr(begin, end - begin); - if (!part.empty()) - out.push_back(part); - - begin = end + 1; - end = uri.find_first_of(delim, begin); - } - - // Add the last segment of the URI (if any) after the last '/' - std::string last_part = uri.substr(begin); - if (!last_part.empty()) { - out.push_back(last_part); - } - - return out; -} - -void -common::parse_path_from_uri(std::string_view uri, - std::string& bucket_name, - std::string& path) -{ - auto parts = split_uri(uri.data()); - EXPECT(parts.size() > 2, "Invalid URI: %s", uri.data()); - - bucket_name = parts[2]; - path = ""; - for (size_t i = 3; i < parts.size(); ++i) { - path += parts[i]; - if (i < parts.size() - 1) { - path += "/"; - } - } -} - -bool -common::is_web_uri(std::string_view uri) -{ - return uri.starts_with("s3://") || uri.starts_with("http://") || - uri.starts_with("https://"); -} - -#ifndef NO_UNIT_TESTS -#ifdef _WIN32 -#define acquire_export __declspec(dllexport) -#else -#define acquire_export -#endif - -extern "C" -{ - acquire_export int unit_test__shard_index_for_chunk() - { - int retval = 0; - try { - std::vector dims; - dims.emplace_back("x", - DimensionType_Space, - 64, - 16, // 64 / 16 = 4 chunks - 2); // 4 / 2 = 2 shards - dims.emplace_back("y", - DimensionType_Space, - 48, - 16, // 48 / 16 = 3 chunks - 1); // 3 / 1 = 3 shards - dims.emplace_back("z", - DimensionType_Space, - 6, - 2, // 6 / 2 = 3 chunks - 1); // 3 / 1 = 3 shards - dims.emplace_back("c", - DimensionType_Channel, - 8, - 4, // 8 / 4 = 2 chunks - 2); // 4 / 2 = 2 shards - dims.emplace_back("t", - DimensionType_Time, - 0, - 5, // 5 timepoints / chunk - 2); // 2 chunks / shard - - CHECK(common::shard_index_for_chunk(0, dims) == 0); - CHECK(common::shard_index_for_chunk(1, dims) == 0); - CHECK(common::shard_index_for_chunk(2, dims) == 1); - CHECK(common::shard_index_for_chunk(3, dims) == 1); - CHECK(common::shard_index_for_chunk(4, dims) == 2); - CHECK(common::shard_index_for_chunk(5, dims) == 2); - CHECK(common::shard_index_for_chunk(6, dims) == 3); - CHECK(common::shard_index_for_chunk(7, dims) == 3); - CHECK(common::shard_index_for_chunk(8, dims) == 4); - CHECK(common::shard_index_for_chunk(9, dims) == 4); - CHECK(common::shard_index_for_chunk(10, dims) == 5); - CHECK(common::shard_index_for_chunk(11, dims) == 5); - CHECK(common::shard_index_for_chunk(12, dims) == 6); - CHECK(common::shard_index_for_chunk(13, dims) == 6); - CHECK(common::shard_index_for_chunk(14, dims) == 7); - CHECK(common::shard_index_for_chunk(15, dims) == 7); - CHECK(common::shard_index_for_chunk(16, dims) == 8); - CHECK(common::shard_index_for_chunk(17, dims) == 8); - CHECK(common::shard_index_for_chunk(18, dims) == 9); - CHECK(common::shard_index_for_chunk(19, dims) == 9); - CHECK(common::shard_index_for_chunk(20, dims) == 10); - CHECK(common::shard_index_for_chunk(21, dims) == 10); - CHECK(common::shard_index_for_chunk(22, dims) == 11); - CHECK(common::shard_index_for_chunk(23, dims) == 11); - CHECK(common::shard_index_for_chunk(24, dims) == 12); - CHECK(common::shard_index_for_chunk(25, dims) == 12); - CHECK(common::shard_index_for_chunk(26, dims) == 13); - CHECK(common::shard_index_for_chunk(27, dims) == 13); - CHECK(common::shard_index_for_chunk(28, dims) == 14); - CHECK(common::shard_index_for_chunk(29, dims) == 14); - CHECK(common::shard_index_for_chunk(30, dims) == 15); - CHECK(common::shard_index_for_chunk(31, dims) == 15); - CHECK(common::shard_index_for_chunk(32, dims) == 16); - CHECK(common::shard_index_for_chunk(33, dims) == 16); - CHECK(common::shard_index_for_chunk(34, dims) == 17); - CHECK(common::shard_index_for_chunk(35, dims) == 17); - CHECK(common::shard_index_for_chunk(36, dims) == 0); - CHECK(common::shard_index_for_chunk(37, dims) == 0); - CHECK(common::shard_index_for_chunk(38, dims) == 1); - CHECK(common::shard_index_for_chunk(39, dims) == 1); - CHECK(common::shard_index_for_chunk(40, dims) == 2); - CHECK(common::shard_index_for_chunk(41, dims) == 2); - CHECK(common::shard_index_for_chunk(42, dims) == 3); - CHECK(common::shard_index_for_chunk(43, dims) == 3); - CHECK(common::shard_index_for_chunk(44, dims) == 4); - CHECK(common::shard_index_for_chunk(45, dims) == 4); - CHECK(common::shard_index_for_chunk(46, dims) == 5); - CHECK(common::shard_index_for_chunk(47, dims) == 5); - CHECK(common::shard_index_for_chunk(48, dims) == 6); - CHECK(common::shard_index_for_chunk(49, dims) == 6); - CHECK(common::shard_index_for_chunk(50, dims) == 7); - CHECK(common::shard_index_for_chunk(51, dims) == 7); - CHECK(common::shard_index_for_chunk(52, dims) == 8); - CHECK(common::shard_index_for_chunk(53, dims) == 8); - CHECK(common::shard_index_for_chunk(54, dims) == 9); - CHECK(common::shard_index_for_chunk(55, dims) == 9); - CHECK(common::shard_index_for_chunk(56, dims) == 10); - CHECK(common::shard_index_for_chunk(57, dims) == 10); - CHECK(common::shard_index_for_chunk(58, dims) == 11); - CHECK(common::shard_index_for_chunk(59, dims) == 11); - CHECK(common::shard_index_for_chunk(60, dims) == 12); - CHECK(common::shard_index_for_chunk(61, dims) == 12); - CHECK(common::shard_index_for_chunk(62, dims) == 13); - CHECK(common::shard_index_for_chunk(63, dims) == 13); - CHECK(common::shard_index_for_chunk(64, dims) == 14); - CHECK(common::shard_index_for_chunk(65, dims) == 14); - CHECK(common::shard_index_for_chunk(66, dims) == 15); - CHECK(common::shard_index_for_chunk(67, dims) == 15); - CHECK(common::shard_index_for_chunk(68, dims) == 16); - CHECK(common::shard_index_for_chunk(69, dims) == 16); - CHECK(common::shard_index_for_chunk(70, dims) == 17); - CHECK(common::shard_index_for_chunk(71, dims) == 17); - CHECK(common::shard_index_for_chunk(72, dims) == 0); - CHECK(common::shard_index_for_chunk(73, dims) == 0); - CHECK(common::shard_index_for_chunk(74, dims) == 1); - CHECK(common::shard_index_for_chunk(75, dims) == 1); - CHECK(common::shard_index_for_chunk(76, dims) == 2); - CHECK(common::shard_index_for_chunk(77, dims) == 2); - CHECK(common::shard_index_for_chunk(78, dims) == 3); - CHECK(common::shard_index_for_chunk(79, dims) == 3); - CHECK(common::shard_index_for_chunk(80, dims) == 4); - CHECK(common::shard_index_for_chunk(81, dims) == 4); - CHECK(common::shard_index_for_chunk(82, dims) == 5); - CHECK(common::shard_index_for_chunk(83, dims) == 5); - CHECK(common::shard_index_for_chunk(84, dims) == 6); - CHECK(common::shard_index_for_chunk(85, dims) == 6); - CHECK(common::shard_index_for_chunk(86, dims) == 7); - CHECK(common::shard_index_for_chunk(87, dims) == 7); - CHECK(common::shard_index_for_chunk(88, dims) == 8); - CHECK(common::shard_index_for_chunk(89, dims) == 8); - CHECK(common::shard_index_for_chunk(90, dims) == 9); - CHECK(common::shard_index_for_chunk(91, dims) == 9); - CHECK(common::shard_index_for_chunk(92, dims) == 10); - CHECK(common::shard_index_for_chunk(93, dims) == 10); - CHECK(common::shard_index_for_chunk(94, dims) == 11); - CHECK(common::shard_index_for_chunk(95, dims) == 11); - CHECK(common::shard_index_for_chunk(96, dims) == 12); - CHECK(common::shard_index_for_chunk(97, dims) == 12); - CHECK(common::shard_index_for_chunk(98, dims) == 13); - CHECK(common::shard_index_for_chunk(99, dims) == 13); - CHECK(common::shard_index_for_chunk(100, dims) == 14); - CHECK(common::shard_index_for_chunk(101, dims) == 14); - CHECK(common::shard_index_for_chunk(102, dims) == 15); - CHECK(common::shard_index_for_chunk(103, dims) == 15); - CHECK(common::shard_index_for_chunk(104, dims) == 16); - CHECK(common::shard_index_for_chunk(105, dims) == 16); - CHECK(common::shard_index_for_chunk(106, dims) == 17); - CHECK(common::shard_index_for_chunk(107, dims) == 17); - CHECK(common::shard_index_for_chunk(108, dims) == 0); - CHECK(common::shard_index_for_chunk(109, dims) == 0); - CHECK(common::shard_index_for_chunk(110, dims) == 1); - CHECK(common::shard_index_for_chunk(111, dims) == 1); - CHECK(common::shard_index_for_chunk(112, dims) == 2); - CHECK(common::shard_index_for_chunk(113, dims) == 2); - CHECK(common::shard_index_for_chunk(114, dims) == 3); - CHECK(common::shard_index_for_chunk(115, dims) == 3); - CHECK(common::shard_index_for_chunk(116, dims) == 4); - CHECK(common::shard_index_for_chunk(117, dims) == 4); - CHECK(common::shard_index_for_chunk(118, dims) == 5); - CHECK(common::shard_index_for_chunk(119, dims) == 5); - CHECK(common::shard_index_for_chunk(120, dims) == 6); - CHECK(common::shard_index_for_chunk(121, dims) == 6); - CHECK(common::shard_index_for_chunk(122, dims) == 7); - CHECK(common::shard_index_for_chunk(123, dims) == 7); - CHECK(common::shard_index_for_chunk(124, dims) == 8); - CHECK(common::shard_index_for_chunk(125, dims) == 8); - CHECK(common::shard_index_for_chunk(126, dims) == 9); - CHECK(common::shard_index_for_chunk(127, dims) == 9); - CHECK(common::shard_index_for_chunk(128, dims) == 10); - CHECK(common::shard_index_for_chunk(129, dims) == 10); - CHECK(common::shard_index_for_chunk(130, dims) == 11); - CHECK(common::shard_index_for_chunk(131, dims) == 11); - CHECK(common::shard_index_for_chunk(132, dims) == 12); - CHECK(common::shard_index_for_chunk(133, dims) == 12); - CHECK(common::shard_index_for_chunk(134, dims) == 13); - CHECK(common::shard_index_for_chunk(135, dims) == 13); - CHECK(common::shard_index_for_chunk(136, dims) == 14); - CHECK(common::shard_index_for_chunk(137, dims) == 14); - CHECK(common::shard_index_for_chunk(138, dims) == 15); - CHECK(common::shard_index_for_chunk(139, dims) == 15); - CHECK(common::shard_index_for_chunk(140, dims) == 16); - CHECK(common::shard_index_for_chunk(141, dims) == 16); - CHECK(common::shard_index_for_chunk(142, dims) == 17); - CHECK(common::shard_index_for_chunk(143, dims) == 17); - - retval = 1; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - - return retval; - } - - acquire_export int unit_test__shard_internal_index() - { - int retval = 0; - - std::vector dims; - dims.emplace_back("x", - DimensionType_Space, - 1080, - 270, // 4 chunks - 3); // 2 ragged shards - dims.emplace_back("y", - DimensionType_Space, - 960, - 320, // 3 chunks - 2); // 2 ragged shards - dims.emplace_back("t", - DimensionType_Time, - 0, - 32, // 32 timepoints / chunk - 1); // 1 shard - - try { - CHECK(common::shard_index_for_chunk(0, dims) == 0); - CHECK(common::shard_internal_index(0, dims) == 0); - - CHECK(common::shard_index_for_chunk(1, dims) == 0); - CHECK(common::shard_internal_index(1, dims) == 1); - - CHECK(common::shard_index_for_chunk(2, dims) == 0); - CHECK(common::shard_internal_index(2, dims) == 2); - - CHECK(common::shard_index_for_chunk(3, dims) == 1); - CHECK(common::shard_internal_index(3, dims) == 0); - - CHECK(common::shard_index_for_chunk(4, dims) == 0); - CHECK(common::shard_internal_index(4, dims) == 3); - - CHECK(common::shard_index_for_chunk(5, dims) == 0); - CHECK(common::shard_internal_index(5, dims) == 4); - - CHECK(common::shard_index_for_chunk(6, dims) == 0); - CHECK(common::shard_internal_index(6, dims) == 5); - - CHECK(common::shard_index_for_chunk(7, dims) == 1); - CHECK(common::shard_internal_index(7, dims) == 3); - - CHECK(common::shard_index_for_chunk(8, dims) == 2); - CHECK(common::shard_internal_index(8, dims) == 0); - - CHECK(common::shard_index_for_chunk(9, dims) == 2); - CHECK(common::shard_internal_index(9, dims) == 1); - - CHECK(common::shard_index_for_chunk(10, dims) == 2); - CHECK(common::shard_internal_index(10, dims) == 2); - - CHECK(common::shard_index_for_chunk(11, dims) == 3); - CHECK(common::shard_internal_index(11, dims) == 0); - retval = 1; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - - return retval; - } - - acquire_export int unit_test__split_uri() - { - try { - auto parts = common::split_uri("s3://bucket/key"); - CHECK(parts.size() == 3); - CHECK(parts[0] == "s3:"); - CHECK(parts[1] == "bucket"); - CHECK(parts[2] == "key"); - - parts = common::split_uri("s3://bucket/key/"); - CHECK(parts.size() == 3); - CHECK(parts[0] == "s3:"); - CHECK(parts[1] == "bucket"); - CHECK(parts[2] == "key"); - - parts = common::split_uri("s3://bucket/key/with/slashes"); - CHECK(parts.size() == 5); - CHECK(parts[0] == "s3:"); - CHECK(parts[1] == "bucket"); - CHECK(parts[2] == "key"); - CHECK(parts[3] == "with"); - CHECK(parts[4] == "slashes"); - - parts = common::split_uri("s3://bucket"); - CHECK(parts.size() == 2); - CHECK(parts[0] == "s3:"); - CHECK(parts[1] == "bucket"); - - parts = common::split_uri("s3://"); - CHECK(parts.size() == 1); - CHECK(parts[0] == "s3:"); - - parts = common::split_uri("s3:///"); - CHECK(parts.size() == 1); - CHECK(parts[0] == "s3:"); - - parts = common::split_uri("s3://bucket/"); - CHECK(parts.size() == 2); - CHECK(parts[0] == "s3:"); - CHECK(parts[1] == "bucket"); - - parts = common::split_uri("s3://bucket/"); - CHECK(parts.size() == 2); - CHECK(parts[0] == "s3:"); - CHECK(parts[1] == "bucket"); - - parts = common::split_uri("s3://bucket/key/with/slashes/"); - CHECK(parts.size() == 5); - CHECK(parts[0] == "s3:"); - CHECK(parts[1] == "bucket"); - CHECK(parts[2] == "key"); - CHECK(parts[3] == "with"); - CHECK(parts[4] == "slashes"); - - parts = common::split_uri("s3://bucket/key/with/slashes//"); - CHECK(parts.size() == 5); - CHECK(parts[0] == "s3:"); - CHECK(parts[1] == "bucket"); - CHECK(parts[2] == "key"); - CHECK(parts[3] == "with"); - CHECK(parts[4] == "slashes"); - return 1; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - - return 0; - } -} -#endif diff --git a/src/driver/common/utilities.hh b/src/driver/common/utilities.hh deleted file mode 100644 index e2b4c8a5..00000000 --- a/src/driver/common/utilities.hh +++ /dev/null @@ -1,124 +0,0 @@ -#ifndef H_ACQUIRE_STORAGE_ZARR_COMMON_UTILITIES_V0 -#define H_ACQUIRE_STORAGE_ZARR_COMMON_UTILITIES_V0 - -#include "logger.h" -#include "device/props/components.h" -#include "device/props/storage.h" -#include "macros.hh" -#include "dimension.hh" - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace fs = std::filesystem; - -namespace acquire::sink::zarr { - -struct Zarr; - -enum class ZarrVersion -{ - V2 = 2, - V3 -}; - -namespace common { -/// @brief Get the number of chunks along a dimension. -/// @param dimension A dimension. -/// @return The number of, possibly ragged, chunks along the dimension, given -/// the dimension's array and chunk sizes. -size_t -chunks_along_dimension(const Dimension& dimension); - -/// @brief Get the number of chunks to hold in memory. -/// @param dimensions The dimensions of the array. -/// @return The number of chunks to buffer before writing out. -size_t -number_of_chunks_in_memory(const std::vector& dimensions); - -/// @brief Get the number of shards along a dimension. -/// @param dimension A dimension. -/// @return The number of shards along the dimension, given the dimension's -/// array, chunk, and shard sizes. -size_t -shards_along_dimension(const Dimension& dimension); - -/// @brief Get the number of shards to write at one time. -/// @param dimensions The dimensions of the array. -/// @return The number of shards to buffer and write out. -size_t -number_of_shards(const std::vector& dimensions); - -/// @brief Get the number of chunks in a single shard. -/// @param dimensions The dimensions of the array. -/// @return The number of chunks in a shard. -size_t -chunks_per_shard(const std::vector& dimensions); - -/// @brief Get the shard index for a given chunk index, given array dimensions. -/// @param chunk_index The index of the chunk. -/// @param dimensions The dimensions of the array. -/// @return The index of the shard containing the chunk. -size_t -shard_index_for_chunk(size_t chunk_index, - const std::vector& dimensions); - -/// @brief Get the internal index of a chunk within a shard. -/// @param chunk_index The index of the chunk. -/// @param dimensions The dimensions of the array. -/// @return The index of the chunk within the shard. -size_t -shard_internal_index(size_t chunk_index, - const std::vector& dimensions); - -/// @brief Get the size, in bytes, of a single chunk. -/// @param dimensions The dimensions of the array. -/// @param dtype The pixel type of the array. -/// @return The number of bytes to allocate for a chunk. -size_t -bytes_per_chunk(const std::vector& dimensions, - const SampleType& dtype); - -/// @brief Get a string representation of the SampleType enum. -/// @param t An enumerated sample type. -/// @return A human-readable representation of the SampleType @par t. -const char* -sample_type_to_string(SampleType t) noexcept; - -/// @brief Align a size to a given alignment. -/// @param n Size to align. -/// @param align Alignment. -/// @return Aligned size. -size_t -align_up(size_t n, size_t align); - -/// @brief Split a URI by the '/' delimiter. -/// @param uri String to split. -/// @return Vector of strings. -std::vector -split_uri(const std::string& uri); - -/// @brief Get the endpoint and bucket name from a URI. -/// @param[in] uri String to parse. -/// @param[out] endpoint The endpoint of the URI. -/// @param[out] bucket_name The bucket name of the URI. -void -parse_path_from_uri(std::string_view uri, - std::string& bucket_name, - std::string& path); - -/// @brief Check if a URI is an S3 URI. -/// @param uri String to check. -/// @return True if the URI is an S3 URI, false otherwise. -bool -is_web_uri(std::string_view uri); -} // namespace acquire::sink::zarr::common -} // namespace acquire::sink::zarr - -#endif // H_ACQUIRE_STORAGE_ZARR_COMMON_UTILITIES_V0 diff --git a/src/driver/writers/array.writer.cpp b/src/driver/writers/array.writer.cpp deleted file mode 100644 index 56e17158..00000000 --- a/src/driver/writers/array.writer.cpp +++ /dev/null @@ -1,975 +0,0 @@ -#include "array.writer.hh" -#include "common/utilities.hh" - -#include -#include -#include -#include - -namespace zarr = acquire::sink::zarr; - -namespace { -/// Returns the index of the chunk in the lattice of chunks for the given frame -/// and dimension. -size_t -chunk_lattice_index(size_t frame_id, - size_t dimension_idx, - const std::vector& dims) -{ - CHECK(dimension_idx >= 2 && dimension_idx < dims.size()); - - // the last dimension is a special case - if (dimension_idx == dims.size() - 1) { - size_t divisor = dims.back().chunk_size_px; - for (auto i = 2; i < dims.size() - 1; ++i) { - const auto& dim = dims.at(i); - divisor *= dim.array_size_px; - } - - CHECK(divisor); - return frame_id / divisor; - } - - size_t mod_divisor = 1, div_divisor = 1; - for (auto i = 2; i <= dimension_idx; ++i) { - const auto& dim = dims.at(i); - mod_divisor *= dim.array_size_px; - div_divisor *= - (i < dimension_idx ? dim.array_size_px : dim.chunk_size_px); - } - - CHECK(mod_divisor); - CHECK(div_divisor); - - return (frame_id % mod_divisor) / div_divisor; -} - -/// Find the offset in the array of chunks for the given frame. -size_t -tile_group_offset(size_t frame_id, const std::vector& dims) -{ - std::vector strides; - strides.push_back(1); - for (auto i = 0; i < dims.size() - 1; ++i) { - const auto& dim = dims.at(i); - CHECK(dim.chunk_size_px); - const auto a = dim.array_size_px, c = dim.chunk_size_px; - strides.push_back(strides.back() * ((a + c - 1) / c)); - } - - size_t offset = 0; - for (auto i = 2; i < dims.size() - 1; ++i) { - const auto idx = chunk_lattice_index(frame_id, i, dims); - const auto stride = strides.at(i); - offset += idx * stride; - } - - return offset; -} - -/// Find the offset inside a chunk for the given frame. -size_t -chunk_internal_offset(size_t frame_id, - const std::vector& dims, - SampleType type) -{ - const auto tile_size = - bytes_of_type(type) * dims.at(0).chunk_size_px * dims.at(1).chunk_size_px; - auto offset = 0; - std::vector array_strides, chunk_strides; - array_strides.push_back(1); - chunk_strides.push_back(1); - for (auto i = 2; i < dims.size(); ++i) { - const auto& dim = dims.at(i); - - if (i < dims.size() - 1) { - CHECK(dim.array_size_px); - } - CHECK(dim.chunk_size_px); - CHECK(array_strides.back()); - - const auto internal_idx = - i == dims.size() - 1 - ? (frame_id / array_strides.back()) % dim.chunk_size_px - : (frame_id / array_strides.back()) % dim.array_size_px % - dim.chunk_size_px; - offset += internal_idx * chunk_strides.back(); - - array_strides.push_back(array_strides.back() * dim.array_size_px); - chunk_strides.push_back(chunk_strides.back() * dim.chunk_size_px); - } - - return offset * tile_size; -} -} // end ::{anonymous} namespace - -bool -zarr::downsample(const ArrayWriterConfig& config, - ArrayWriterConfig& downsampled_config) -{ - // downsample dimensions - downsampled_config.dimensions.clear(); - for (const auto& dim : config.dimensions) { - if (dim.kind == DimensionType_Channel) { // don't downsample channels - downsampled_config.dimensions.push_back(dim); - } else { - const uint32_t array_size_px = - (dim.array_size_px + (dim.array_size_px % 2)) / 2; - - const uint32_t chunk_size_px = - dim.array_size_px == 0 - ? dim.chunk_size_px - : std::min(dim.chunk_size_px, array_size_px); - - CHECK(chunk_size_px); - const uint32_t n_chunks = - (array_size_px + chunk_size_px - 1) / chunk_size_px; - - const uint32_t shard_size_chunks = - dim.array_size_px == 0 - ? 1 - : std::min(n_chunks, dim.shard_size_chunks); - - downsampled_config.dimensions.emplace_back(dim.name, - dim.kind, - array_size_px, - chunk_size_px, - shard_size_chunks); - } - } - - // downsample image_shape - downsampled_config.image_shape = config.image_shape; - - downsampled_config.image_shape.dims.width = - downsampled_config.dimensions.at(0).array_size_px; - downsampled_config.image_shape.dims.height = - downsampled_config.dimensions.at(1).array_size_px; - - downsampled_config.image_shape.strides.height = - downsampled_config.image_shape.dims.width; - downsampled_config.image_shape.strides.planes = - downsampled_config.image_shape.dims.width * - downsampled_config.image_shape.dims.height; - - downsampled_config.level_of_detail = config.level_of_detail + 1; - downsampled_config.dataset_root = config.dataset_root; - - // copy the Blosc compression parameters - downsampled_config.compression_params = config.compression_params; - - // can we downsample downsampled_config? - for (auto i = 0; i < config.dimensions.size(); ++i) { - // downsampling made the chunk size strictly smaller - const auto& dim = config.dimensions.at(i); - const auto& downsampled_dim = downsampled_config.dimensions.at(i); - - if (dim.chunk_size_px > downsampled_dim.chunk_size_px) { - return false; - } - } - - return true; -} - -/// Writer -zarr::ArrayWriter::ArrayWriter( - const ArrayWriterConfig& config, - std::shared_ptr thread_pool, - std::shared_ptr connection_pool) - : config_{ config } - , thread_pool_{ thread_pool } - , connection_pool_{ connection_pool } - , bytes_to_flush_{ 0 } - , frames_written_{ 0 } - , append_chunk_index_{ 0 } - , is_finalizing_{ false } -{ -} - -size_t -zarr::ArrayWriter::write(const uint8_t* data, size_t bytes_of_frame) -{ - if (bytes_of_image(&config_.image_shape) != bytes_of_frame) { - return 0; - } - - if (chunk_buffers_.empty()) { - make_buffers_(); - } - - // split the incoming frame into tiles and write them to the chunk buffers - const auto bytes_written = write_frame_to_chunks_(data, bytes_of_frame); - bytes_to_flush_ += bytes_written; - ++frames_written_; - - if (should_flush_()) { - flush_(); - } - - return bytes_written; -} - -void -zarr::ArrayWriter::finalize() -{ - is_finalizing_ = true; - flush_(); - close_sinks_(); - is_finalizing_ = false; -} - -void -zarr::ArrayWriter::make_buffers_() noexcept -{ - const size_t n_chunks = - common::number_of_chunks_in_memory(config_.dimensions); - chunk_buffers_.resize(n_chunks); // no-op if already the correct size - - const auto bytes_per_chunk = - common::bytes_per_chunk(config_.dimensions, config_.image_shape.type); - - for (auto& buf : chunk_buffers_) { - buf.resize(bytes_per_chunk); - std::fill_n(buf.begin(), bytes_per_chunk, 0); - } -} - -size_t -zarr::ArrayWriter::write_frame_to_chunks_(const uint8_t* buf, size_t buf_size) -{ - // break the frame into tiles and write them to the chunk buffers - const auto image_shape = config_.image_shape; - const auto bytes_per_px = bytes_of_type(image_shape.type); - - const auto frame_cols = image_shape.dims.width; - const auto frame_rows = image_shape.dims.height; - - const auto& dimensions = config_.dimensions; - const auto tile_cols = dimensions.at(0).chunk_size_px; - const auto tile_rows = dimensions.at(1).chunk_size_px; - - if (tile_cols == 0 || tile_rows == 0) { - return 0; - } - - const auto bytes_per_row = tile_cols * bytes_per_px; - - size_t bytes_written = 0; - - const auto n_tiles_x = (frame_cols + tile_cols - 1) / tile_cols; - const auto n_tiles_y = (frame_rows + tile_rows - 1) / tile_rows; - - // don't take the frame id from the incoming frame, as the camera may have - // dropped frames - const auto frame_id = frames_written_; - - // offset among the chunks in the lattice - const auto group_offset = tile_group_offset(frame_id, dimensions); - // offset within the chunk - const auto chunk_offset = - chunk_internal_offset(frame_id, dimensions, image_shape.type); - - for (auto i = 0; i < n_tiles_y; ++i) { - // TODO (aliddell): we can optimize this when tiles_per_frame_x_ is 1 - for (auto j = 0; j < n_tiles_x; ++j) { - const auto c = group_offset + i * n_tiles_x + j; - auto& chunk = chunk_buffers_.at(c); - auto chunk_it = - chunk.begin() + static_cast(chunk_offset); - - for (auto k = 0; k < tile_rows; ++k) { - const auto frame_row = i * tile_rows + k; - if (frame_row < frame_rows) { - const auto frame_col = j * tile_cols; - - const auto region_width = - std::min(frame_col + tile_cols, frame_cols) - frame_col; - - const auto region_start = - bytes_per_px * (frame_row * frame_cols + frame_col); - const auto nbytes = region_width * bytes_per_px; - const auto region_stop = region_start + nbytes; - if (region_stop > buf_size) { - LOGE("Buffer overflow"); - return bytes_written; - } - - // copy region - if (nbytes > std::distance(chunk_it, chunk.end())) { - LOGE("Buffer overflow"); - return bytes_written; - } - std::copy(buf + region_start, buf + region_stop, chunk_it); - - bytes_written += (region_stop - region_start); - } - chunk_it += static_cast(bytes_per_row); - } - } - } - - return bytes_written; -} - -bool -zarr::ArrayWriter::should_flush_() const -{ - const auto& dims = config_.dimensions; - size_t frames_before_flush = dims.back().chunk_size_px; - for (auto i = 2; i < dims.size() - 1; ++i) { - frames_before_flush *= dims.at(i).array_size_px; - } - - CHECK(frames_before_flush > 0); - return frames_written_ % frames_before_flush == 0; -} - -void -zarr::ArrayWriter::compress_buffers_() noexcept -{ - if (!config_.compression_params.has_value()) { - return; - } - - TRACE("Compressing"); - - BloscCompressionParams params = config_.compression_params.value(); - const auto bytes_per_px = bytes_of_type(config_.image_shape.type); - - std::scoped_lock lock(buffers_mutex_); - std::latch latch(chunk_buffers_.size()); - for (auto i = 0; i < chunk_buffers_.size(); ++i) { - auto& chunk = chunk_buffers_.at(i); - - thread_pool_->push_to_job_queue([¶ms, - buf = &chunk, - bytes_per_px, - &latch](std::string& err) -> bool { - bool success = false; - const size_t bytes_of_chunk = buf->size(); - - try { - const auto tmp_size = bytes_of_chunk + BLOSC_MAX_OVERHEAD; - std::vector tmp(tmp_size); - const auto nb = - blosc_compress_ctx(params.clevel, - params.shuffle, - bytes_per_px, - bytes_of_chunk, - buf->data(), - tmp.data(), - tmp_size, - params.codec_id.c_str(), - 0 /* blocksize - 0:automatic */, - 1); - - tmp.resize(nb); - buf->swap(tmp); - - success = true; - } catch (const std::exception& exc) { - char msg[128]; - snprintf( - msg, sizeof(msg), "Failed to compress chunk: %s", exc.what()); - err = msg; - } catch (...) { - err = "Failed to compress chunk (unknown)"; - } - latch.count_down(); - - return success; - }); - } - - // wait for all threads to finish - latch.wait(); -} - -void -zarr::ArrayWriter::flush_() -{ - if (bytes_to_flush_ == 0) { - return; - } - - // compress buffers and write out - compress_buffers_(); - CHECK(flush_impl_()); - - const auto should_rollover = should_rollover_(); - if (should_rollover) { - rollover_(); - } - - if (should_rollover || is_finalizing_) { - CHECK(write_array_metadata_()); - } - - // reset buffers - make_buffers_(); - - // reset state - bytes_to_flush_ = 0; -} - -void -zarr::ArrayWriter::close_sinks_() -{ - data_sinks_.clear(); -} - -void -zarr::ArrayWriter::rollover_() -{ - TRACE("Rolling over"); - - close_sinks_(); - ++append_chunk_index_; -} - -#ifndef NO_UNIT_TESTS -#ifdef _WIN32 -#define acquire_export __declspec(dllexport) -#else -#define acquire_export -#endif - -namespace common = zarr::common; - -class TestWriter : public zarr::ArrayWriter -{ - public: - TestWriter(const zarr::ArrayWriterConfig& array_spec, - std::shared_ptr thread_pool) - : zarr::ArrayWriter(array_spec, thread_pool, nullptr) - { - } - - private: - bool should_rollover_() const override { return false; } - bool flush_impl_() override { return true; } - bool write_array_metadata_() override { return true; } -}; - -extern "C" -{ - acquire_export int unit_test__chunk_lattice_index() - { - int retval = 0; - try { - std::vector dims; - dims.emplace_back("x", DimensionType_Space, 64, 16, 0); // 4 chunks - dims.emplace_back("y", DimensionType_Space, 48, 16, 0); // 3 chunks - dims.emplace_back("z", DimensionType_Space, 5, 2, 0); // 3 chunks - dims.emplace_back("c", DimensionType_Channel, 3, 2, 0); // 2 chunks - dims.emplace_back( - "t", DimensionType_Time, 0, 5, 0); // 5 timepoints / chunk - - CHECK(chunk_lattice_index(0, 2, dims) == 0); - CHECK(chunk_lattice_index(0, 3, dims) == 0); - CHECK(chunk_lattice_index(0, 4, dims) == 0); - CHECK(chunk_lattice_index(1, 2, dims) == 0); - CHECK(chunk_lattice_index(1, 3, dims) == 0); - CHECK(chunk_lattice_index(1, 4, dims) == 0); - CHECK(chunk_lattice_index(2, 2, dims) == 1); - CHECK(chunk_lattice_index(2, 3, dims) == 0); - CHECK(chunk_lattice_index(2, 4, dims) == 0); - CHECK(chunk_lattice_index(3, 2, dims) == 1); - CHECK(chunk_lattice_index(3, 3, dims) == 0); - CHECK(chunk_lattice_index(3, 4, dims) == 0); - CHECK(chunk_lattice_index(4, 2, dims) == 2); - CHECK(chunk_lattice_index(4, 3, dims) == 0); - CHECK(chunk_lattice_index(4, 4, dims) == 0); - CHECK(chunk_lattice_index(5, 2, dims) == 0); - CHECK(chunk_lattice_index(5, 3, dims) == 0); - CHECK(chunk_lattice_index(5, 4, dims) == 0); - CHECK(chunk_lattice_index(12, 2, dims) == 1); - CHECK(chunk_lattice_index(12, 3, dims) == 1); - CHECK(chunk_lattice_index(12, 4, dims) == 0); - CHECK(chunk_lattice_index(19, 2, dims) == 2); - CHECK(chunk_lattice_index(19, 3, dims) == 0); - CHECK(chunk_lattice_index(19, 4, dims) == 0); - CHECK(chunk_lattice_index(26, 2, dims) == 0); - CHECK(chunk_lattice_index(26, 3, dims) == 1); - CHECK(chunk_lattice_index(26, 4, dims) == 0); - CHECK(chunk_lattice_index(33, 2, dims) == 1); - CHECK(chunk_lattice_index(33, 3, dims) == 0); - CHECK(chunk_lattice_index(33, 4, dims) == 0); - CHECK(chunk_lattice_index(40, 2, dims) == 0); - CHECK(chunk_lattice_index(40, 3, dims) == 1); - CHECK(chunk_lattice_index(40, 4, dims) == 0); - CHECK(chunk_lattice_index(47, 2, dims) == 1); - CHECK(chunk_lattice_index(47, 3, dims) == 0); - CHECK(chunk_lattice_index(47, 4, dims) == 0); - CHECK(chunk_lattice_index(54, 2, dims) == 2); - CHECK(chunk_lattice_index(54, 3, dims) == 0); - CHECK(chunk_lattice_index(54, 4, dims) == 0); - CHECK(chunk_lattice_index(61, 2, dims) == 0); - CHECK(chunk_lattice_index(61, 3, dims) == 0); - CHECK(chunk_lattice_index(61, 4, dims) == 0); - CHECK(chunk_lattice_index(68, 2, dims) == 1); - CHECK(chunk_lattice_index(68, 3, dims) == 0); - CHECK(chunk_lattice_index(68, 4, dims) == 0); - CHECK(chunk_lattice_index(74, 2, dims) == 2); - CHECK(chunk_lattice_index(74, 3, dims) == 1); - CHECK(chunk_lattice_index(74, 4, dims) == 0); - CHECK(chunk_lattice_index(75, 2, dims) == 0); - CHECK(chunk_lattice_index(75, 3, dims) == 0); - CHECK(chunk_lattice_index(75, 4, dims) == 1); - - retval = 1; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - return retval; - } - - acquire_export int unit_test__tile_group_offset() - { - int retval = 0; - - std::vector dims; - dims.emplace_back("x", DimensionType_Space, 64, 16, 0); // 4 chunks - dims.emplace_back("y", DimensionType_Space, 48, 16, 0); // 3 chunks - dims.emplace_back("z", DimensionType_Space, 5, 2, 0); // 3 chunks - dims.emplace_back("c", DimensionType_Channel, 3, 2, 0); // 2 chunks - dims.emplace_back( - "t", DimensionType_Time, 0, 5, 0); // 5 timepoints / chunk - - try { - CHECK(tile_group_offset(0, dims) == 0); - CHECK(tile_group_offset(1, dims) == 0); - CHECK(tile_group_offset(2, dims) == 12); - CHECK(tile_group_offset(3, dims) == 12); - CHECK(tile_group_offset(4, dims) == 24); - CHECK(tile_group_offset(5, dims) == 0); - CHECK(tile_group_offset(6, dims) == 0); - CHECK(tile_group_offset(7, dims) == 12); - CHECK(tile_group_offset(8, dims) == 12); - CHECK(tile_group_offset(9, dims) == 24); - CHECK(tile_group_offset(10, dims) == 36); - CHECK(tile_group_offset(11, dims) == 36); - CHECK(tile_group_offset(12, dims) == 48); - CHECK(tile_group_offset(13, dims) == 48); - CHECK(tile_group_offset(14, dims) == 60); - CHECK(tile_group_offset(15, dims) == 0); - CHECK(tile_group_offset(16, dims) == 0); - CHECK(tile_group_offset(17, dims) == 12); - CHECK(tile_group_offset(18, dims) == 12); - CHECK(tile_group_offset(19, dims) == 24); - CHECK(tile_group_offset(20, dims) == 0); - CHECK(tile_group_offset(21, dims) == 0); - CHECK(tile_group_offset(22, dims) == 12); - CHECK(tile_group_offset(23, dims) == 12); - CHECK(tile_group_offset(24, dims) == 24); - CHECK(tile_group_offset(25, dims) == 36); - CHECK(tile_group_offset(26, dims) == 36); - CHECK(tile_group_offset(27, dims) == 48); - CHECK(tile_group_offset(28, dims) == 48); - CHECK(tile_group_offset(29, dims) == 60); - CHECK(tile_group_offset(30, dims) == 0); - CHECK(tile_group_offset(31, dims) == 0); - CHECK(tile_group_offset(32, dims) == 12); - CHECK(tile_group_offset(33, dims) == 12); - CHECK(tile_group_offset(34, dims) == 24); - CHECK(tile_group_offset(35, dims) == 0); - CHECK(tile_group_offset(36, dims) == 0); - CHECK(tile_group_offset(37, dims) == 12); - CHECK(tile_group_offset(38, dims) == 12); - CHECK(tile_group_offset(39, dims) == 24); - CHECK(tile_group_offset(40, dims) == 36); - CHECK(tile_group_offset(41, dims) == 36); - CHECK(tile_group_offset(42, dims) == 48); - CHECK(tile_group_offset(43, dims) == 48); - CHECK(tile_group_offset(44, dims) == 60); - CHECK(tile_group_offset(45, dims) == 0); - CHECK(tile_group_offset(46, dims) == 0); - CHECK(tile_group_offset(47, dims) == 12); - CHECK(tile_group_offset(48, dims) == 12); - CHECK(tile_group_offset(49, dims) == 24); - CHECK(tile_group_offset(50, dims) == 0); - CHECK(tile_group_offset(51, dims) == 0); - CHECK(tile_group_offset(52, dims) == 12); - CHECK(tile_group_offset(53, dims) == 12); - CHECK(tile_group_offset(54, dims) == 24); - CHECK(tile_group_offset(55, dims) == 36); - CHECK(tile_group_offset(56, dims) == 36); - CHECK(tile_group_offset(57, dims) == 48); - CHECK(tile_group_offset(58, dims) == 48); - CHECK(tile_group_offset(59, dims) == 60); - CHECK(tile_group_offset(60, dims) == 0); - CHECK(tile_group_offset(61, dims) == 0); - CHECK(tile_group_offset(62, dims) == 12); - CHECK(tile_group_offset(63, dims) == 12); - CHECK(tile_group_offset(64, dims) == 24); - CHECK(tile_group_offset(65, dims) == 0); - CHECK(tile_group_offset(66, dims) == 0); - CHECK(tile_group_offset(67, dims) == 12); - CHECK(tile_group_offset(68, dims) == 12); - CHECK(tile_group_offset(69, dims) == 24); - CHECK(tile_group_offset(70, dims) == 36); - CHECK(tile_group_offset(71, dims) == 36); - CHECK(tile_group_offset(72, dims) == 48); - CHECK(tile_group_offset(73, dims) == 48); - CHECK(tile_group_offset(74, dims) == 60); - CHECK(tile_group_offset(75, dims) == 0); - - retval = 1; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - - return retval; - } - - acquire_export int unit_test__chunk_internal_offset() - { - int retval = 0; - - std::vector dims; - dims.emplace_back("x", DimensionType_Space, 64, 16, 0); // 4 chunks - dims.emplace_back("y", DimensionType_Space, 48, 16, 0); // 3 chunks - dims.emplace_back("z", DimensionType_Space, 5, 2, 0); // 3 chunks - dims.emplace_back("c", DimensionType_Channel, 3, 2, 0); // 2 chunks - dims.emplace_back( - "t", DimensionType_Time, 0, 5, 0); // 5 timepoints / chunk - - try { - CHECK(chunk_internal_offset(0, dims, SampleType_u16) == 0); - CHECK(chunk_internal_offset(1, dims, SampleType_u16) == 512); - CHECK(chunk_internal_offset(2, dims, SampleType_u16) == 0); - CHECK(chunk_internal_offset(3, dims, SampleType_u16) == 512); - CHECK(chunk_internal_offset(4, dims, SampleType_u16) == 0); - CHECK(chunk_internal_offset(5, dims, SampleType_u16) == 1024); - CHECK(chunk_internal_offset(6, dims, SampleType_u16) == 1536); - CHECK(chunk_internal_offset(7, dims, SampleType_u16) == 1024); - CHECK(chunk_internal_offset(8, dims, SampleType_u16) == 1536); - CHECK(chunk_internal_offset(9, dims, SampleType_u16) == 1024); - CHECK(chunk_internal_offset(10, dims, SampleType_u16) == 0); - CHECK(chunk_internal_offset(11, dims, SampleType_u16) == 512); - CHECK(chunk_internal_offset(12, dims, SampleType_u16) == 0); - CHECK(chunk_internal_offset(13, dims, SampleType_u16) == 512); - CHECK(chunk_internal_offset(14, dims, SampleType_u16) == 0); - CHECK(chunk_internal_offset(15, dims, SampleType_u16) == 2048); - CHECK(chunk_internal_offset(16, dims, SampleType_u16) == 2560); - CHECK(chunk_internal_offset(17, dims, SampleType_u16) == 2048); - CHECK(chunk_internal_offset(18, dims, SampleType_u16) == 2560); - CHECK(chunk_internal_offset(19, dims, SampleType_u16) == 2048); - CHECK(chunk_internal_offset(20, dims, SampleType_u16) == 3072); - CHECK(chunk_internal_offset(21, dims, SampleType_u16) == 3584); - CHECK(chunk_internal_offset(22, dims, SampleType_u16) == 3072); - CHECK(chunk_internal_offset(23, dims, SampleType_u16) == 3584); - CHECK(chunk_internal_offset(24, dims, SampleType_u16) == 3072); - CHECK(chunk_internal_offset(25, dims, SampleType_u16) == 2048); - CHECK(chunk_internal_offset(26, dims, SampleType_u16) == 2560); - CHECK(chunk_internal_offset(27, dims, SampleType_u16) == 2048); - CHECK(chunk_internal_offset(28, dims, SampleType_u16) == 2560); - CHECK(chunk_internal_offset(29, dims, SampleType_u16) == 2048); - CHECK(chunk_internal_offset(30, dims, SampleType_u16) == 4096); - CHECK(chunk_internal_offset(31, dims, SampleType_u16) == 4608); - CHECK(chunk_internal_offset(32, dims, SampleType_u16) == 4096); - CHECK(chunk_internal_offset(33, dims, SampleType_u16) == 4608); - CHECK(chunk_internal_offset(34, dims, SampleType_u16) == 4096); - CHECK(chunk_internal_offset(35, dims, SampleType_u16) == 5120); - CHECK(chunk_internal_offset(36, dims, SampleType_u16) == 5632); - CHECK(chunk_internal_offset(37, dims, SampleType_u16) == 5120); - CHECK(chunk_internal_offset(38, dims, SampleType_u16) == 5632); - CHECK(chunk_internal_offset(39, dims, SampleType_u16) == 5120); - CHECK(chunk_internal_offset(40, dims, SampleType_u16) == 4096); - CHECK(chunk_internal_offset(41, dims, SampleType_u16) == 4608); - CHECK(chunk_internal_offset(42, dims, SampleType_u16) == 4096); - CHECK(chunk_internal_offset(43, dims, SampleType_u16) == 4608); - CHECK(chunk_internal_offset(44, dims, SampleType_u16) == 4096); - CHECK(chunk_internal_offset(45, dims, SampleType_u16) == 6144); - CHECK(chunk_internal_offset(46, dims, SampleType_u16) == 6656); - CHECK(chunk_internal_offset(47, dims, SampleType_u16) == 6144); - CHECK(chunk_internal_offset(48, dims, SampleType_u16) == 6656); - CHECK(chunk_internal_offset(49, dims, SampleType_u16) == 6144); - CHECK(chunk_internal_offset(50, dims, SampleType_u16) == 7168); - CHECK(chunk_internal_offset(51, dims, SampleType_u16) == 7680); - CHECK(chunk_internal_offset(52, dims, SampleType_u16) == 7168); - CHECK(chunk_internal_offset(53, dims, SampleType_u16) == 7680); - CHECK(chunk_internal_offset(54, dims, SampleType_u16) == 7168); - CHECK(chunk_internal_offset(55, dims, SampleType_u16) == 6144); - CHECK(chunk_internal_offset(56, dims, SampleType_u16) == 6656); - CHECK(chunk_internal_offset(57, dims, SampleType_u16) == 6144); - CHECK(chunk_internal_offset(58, dims, SampleType_u16) == 6656); - CHECK(chunk_internal_offset(59, dims, SampleType_u16) == 6144); - CHECK(chunk_internal_offset(60, dims, SampleType_u16) == 8192); - CHECK(chunk_internal_offset(61, dims, SampleType_u16) == 8704); - CHECK(chunk_internal_offset(62, dims, SampleType_u16) == 8192); - CHECK(chunk_internal_offset(63, dims, SampleType_u16) == 8704); - CHECK(chunk_internal_offset(64, dims, SampleType_u16) == 8192); - CHECK(chunk_internal_offset(65, dims, SampleType_u16) == 9216); - CHECK(chunk_internal_offset(66, dims, SampleType_u16) == 9728); - CHECK(chunk_internal_offset(67, dims, SampleType_u16) == 9216); - CHECK(chunk_internal_offset(68, dims, SampleType_u16) == 9728); - CHECK(chunk_internal_offset(69, dims, SampleType_u16) == 9216); - CHECK(chunk_internal_offset(70, dims, SampleType_u16) == 8192); - CHECK(chunk_internal_offset(71, dims, SampleType_u16) == 8704); - CHECK(chunk_internal_offset(72, dims, SampleType_u16) == 8192); - CHECK(chunk_internal_offset(73, dims, SampleType_u16) == 8704); - CHECK(chunk_internal_offset(74, dims, SampleType_u16) == 8192); - CHECK(chunk_internal_offset(75, dims, SampleType_u16) == 0); - - retval = 1; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - - return retval; - } - - acquire_export int unit_test__writer__write_frame_to_chunks() - { - const auto base_dir = fs::temp_directory_path() / "acquire"; - int retval = 0; - - const unsigned int array_width = 64, array_height = 48, - array_planes = 2, array_channels = 1, - array_timepoints = 2; - const unsigned int chunk_width = 16, chunk_height = 16, - chunk_planes = 1, chunk_channels = 1, - chunk_timepoints = 1; - - const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks - const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks - - const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 2 chunks - const unsigned int chunks_in_c = - (array_channels + chunk_channels - 1) / chunk_channels; // 1 chunk - const unsigned int chunks_in_t = - (array_timepoints + chunk_timepoints - 1) / - chunk_timepoints; // 2 chunks - const unsigned int n_frames = - array_planes * array_channels * array_timepoints; - - const ImageShape shape - { - .dims = { - .width = array_width, - .height = array_height, - }, - .strides = { - .width = 1, - .height = array_width, - .planes = array_width * array_height, - }, - .type = SampleType_u16, - }; - const unsigned int nbytes_px = bytes_of_type(shape.type); - - try { - auto thread_pool = std::make_shared( - std::thread::hardware_concurrency(), - [](const std::string& err) { LOGE("Error: %s", err.c_str()); }); - - std::vector dims; - dims.emplace_back( - "x", DimensionType_Space, array_width, chunk_width, 0); - dims.emplace_back( - "y", DimensionType_Space, array_height, chunk_height, 0); - dims.emplace_back( - "z", DimensionType_Space, array_planes, chunk_planes, 0); - dims.emplace_back( - "c", DimensionType_Channel, array_channels, chunk_channels, 0); - dims.emplace_back( - "t", DimensionType_Time, array_timepoints, chunk_timepoints, 0); - - zarr::ArrayWriterConfig config = { - .image_shape = shape, - .dimensions = dims, - .dataset_root = base_dir.string(), - .compression_params = std::nullopt, - }; - - TestWriter writer(config, thread_pool); - - const size_t frame_size = array_width * array_height * nbytes_px; - std::vector data(frame_size, 0); - - for (auto i = 0; i < n_frames; ++i) { - CHECK(writer.write(data.data(), frame_size) == frame_size); - } - - retval = 1; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - - // cleanup - if (fs::exists(base_dir)) { - fs::remove_all(base_dir); - } - return retval; - } - - acquire_export int unit_test__downsample_writer_config() - { - int retval = 0; - try { - const fs::path base_dir = "acquire"; - - zarr::ArrayWriterConfig config { - .image_shape = { - .dims = { - .channels = 1, - .width = 64, - .height = 48, - .planes = 1, - }, - .strides = { - .channels = 1, - .width = 1, - .height = 64, - .planes = 64 * 48 - }, - .type = SampleType_u8 - }, - .dimensions = {}, - .level_of_detail = 0, - .dataset_root = base_dir.string(), - .compression_params = std::nullopt - }; - - config.dimensions.emplace_back( - "x", DimensionType_Space, 64, 16, 2); // 4 chunks, 2 shards - config.dimensions.emplace_back( - "y", DimensionType_Space, 48, 16, 3); // 3 chunks, 1 shard - config.dimensions.emplace_back( - "z", DimensionType_Space, 7, 3, 3); // 3 chunks, 3 shards - config.dimensions.emplace_back( - "c", DimensionType_Channel, 2, 1, 1); // 2 chunks, 2 shards - config.dimensions.emplace_back("t", - DimensionType_Time, - 0, - 5, - 1); // 5 timepoints / chunk, 1 shard - - zarr::ArrayWriterConfig downsampled_config; - CHECK(zarr::downsample(config, downsampled_config)); - - // check dimensions - CHECK(downsampled_config.dimensions.size() == 5); - CHECK(downsampled_config.dimensions.at(0).name == "x"); - CHECK(downsampled_config.dimensions.at(0).array_size_px == 32); - CHECK(downsampled_config.dimensions.at(0).chunk_size_px == 16); - CHECK(downsampled_config.dimensions.at(0).shard_size_chunks == 2); - - CHECK(downsampled_config.dimensions.at(1).name == "y"); - CHECK(downsampled_config.dimensions.at(1).array_size_px == 24); - CHECK(downsampled_config.dimensions.at(1).chunk_size_px == 16); - CHECK(downsampled_config.dimensions.at(1).shard_size_chunks == 2); - - CHECK(downsampled_config.dimensions.at(2).name == "z"); - CHECK(downsampled_config.dimensions.at(2).array_size_px == 4); - CHECK(downsampled_config.dimensions.at(2).chunk_size_px == 3); - CHECK(downsampled_config.dimensions.at(2).shard_size_chunks == 2); - - CHECK(downsampled_config.dimensions.at(3).name == "c"); - // we don't downsample channels - CHECK(downsampled_config.dimensions.at(3).array_size_px == 2); - CHECK(downsampled_config.dimensions.at(3).chunk_size_px == 1); - CHECK(downsampled_config.dimensions.at(3).shard_size_chunks == 1); - - CHECK(downsampled_config.dimensions.at(4).name == "t"); - CHECK(downsampled_config.dimensions.at(4).array_size_px == 0); - CHECK(downsampled_config.dimensions.at(4).chunk_size_px == 5); - CHECK(downsampled_config.dimensions.at(4).shard_size_chunks == 1); - - // check image shape - CHECK(downsampled_config.image_shape.dims.channels == 1); - CHECK(downsampled_config.image_shape.dims.width == 32); - CHECK(downsampled_config.image_shape.dims.height == 24); - CHECK(downsampled_config.image_shape.dims.planes == 1); - - CHECK(downsampled_config.image_shape.strides.channels == 1); - CHECK(downsampled_config.image_shape.strides.width == 1); - CHECK(downsampled_config.image_shape.strides.height == 32); - CHECK(downsampled_config.image_shape.strides.planes == 32 * 24); - - // check level of detail - CHECK(downsampled_config.level_of_detail == 1); - - // check dataset root - CHECK(downsampled_config.dataset_root == config.dataset_root); - - // check compression params - CHECK(!downsampled_config.compression_params.has_value()); - - // downsample again - config = std::move(downsampled_config); - - // can't downsample anymore - CHECK(!zarr::downsample(config, downsampled_config)); - - // check dimensions - CHECK(downsampled_config.dimensions.size() == 5); - CHECK(downsampled_config.dimensions.at(0).name == "x"); - CHECK(downsampled_config.dimensions.at(0).array_size_px == 16); - CHECK(downsampled_config.dimensions.at(0).chunk_size_px == 16); - CHECK(downsampled_config.dimensions.at(0).shard_size_chunks == 1); - - CHECK(downsampled_config.dimensions.at(1).name == "y"); - CHECK(downsampled_config.dimensions.at(1).array_size_px == 12); - CHECK(downsampled_config.dimensions.at(1).chunk_size_px == 12); - CHECK(downsampled_config.dimensions.at(1).shard_size_chunks == 1); - - CHECK(downsampled_config.dimensions.at(2).name == "z"); - CHECK(downsampled_config.dimensions.at(2).array_size_px == 2); - CHECK(downsampled_config.dimensions.at(2).chunk_size_px == 2); - CHECK(downsampled_config.dimensions.at(2).shard_size_chunks == 1); - - CHECK(downsampled_config.dimensions.at(3).name == "c"); - // we don't downsample channels - CHECK(downsampled_config.dimensions.at(3).array_size_px == 2); - CHECK(downsampled_config.dimensions.at(3).chunk_size_px == 1); - CHECK(downsampled_config.dimensions.at(3).shard_size_chunks == 1); - - CHECK(downsampled_config.dimensions.at(4).name == "t"); - CHECK(downsampled_config.dimensions.at(4).array_size_px == 0); - CHECK(downsampled_config.dimensions.at(4).chunk_size_px == 5); - CHECK(downsampled_config.dimensions.at(4).shard_size_chunks == 1); - - // check image shape - CHECK(downsampled_config.image_shape.dims.channels == 1); - CHECK(downsampled_config.image_shape.dims.width == 16); - CHECK(downsampled_config.image_shape.dims.height == 12); - CHECK(downsampled_config.image_shape.dims.planes == 1); - - CHECK(downsampled_config.image_shape.strides.channels == 1); - CHECK(downsampled_config.image_shape.strides.width == 1); - CHECK(downsampled_config.image_shape.strides.height == 16); - CHECK(downsampled_config.image_shape.strides.planes == 16 * 12); - - // check level of detail - CHECK(downsampled_config.level_of_detail == 2); - - // check data root - CHECK(downsampled_config.dataset_root == config.dataset_root); - - // check compression params - CHECK(!downsampled_config.compression_params.has_value()); - - retval = 1; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - return retval; - } -}; -#endif diff --git a/src/driver/writers/array.writer.hh b/src/driver/writers/array.writer.hh deleted file mode 100644 index 27d7219c..00000000 --- a/src/driver/writers/array.writer.hh +++ /dev/null @@ -1,88 +0,0 @@ -#pragma once - -#include "platform.h" -#include "device/props/components.h" - -#include "common/dimension.hh" -#include "common/thread.pool.hh" -#include "common/s3.connection.hh" -#include "blosc.compressor.hh" -#include "file.sink.hh" - -#include -#include - -namespace fs = std::filesystem; - -namespace acquire::sink::zarr { -struct Zarr; - -struct ArrayWriterConfig final -{ - ImageShape image_shape; - std::vector dimensions; - int level_of_detail; - std::string dataset_root; - std::optional compression_params; -}; - -/// @brief Downsample the array writer configuration to a lower resolution. -/// @param[in] config The original array writer configuration. -/// @param[out] downsampled_config The downsampled array writer configuration. -/// @return True if @p downsampled_config can be downsampled further. -/// This is determined by the chunk size in @p config. This function will return -/// false if and only if downsampling brings one or more dimensions lower than -/// the chunk size along that dimension. -[[nodiscard]] bool -downsample(const ArrayWriterConfig& config, - ArrayWriterConfig& downsampled_config); - -struct ArrayWriter -{ - public: - ArrayWriter() = delete; - ArrayWriter(const ArrayWriterConfig& config, - std::shared_ptr thread_pool, - std::shared_ptr connection_pool); - - virtual ~ArrayWriter() noexcept = default; - - [[nodiscard]] size_t write(const uint8_t* data, size_t bytes_of_frame); - void finalize(); - - protected: - ArrayWriterConfig config_; - - /// Chunking - std::vector> chunk_buffers_; - - /// Filesystem - std::string data_root_; - std::string meta_root_; - std::vector> data_sinks_; - std::unique_ptr metadata_sink_; - - /// Multithreading - std::shared_ptr thread_pool_; - std::mutex buffers_mutex_; - - /// Bookkeeping - uint64_t bytes_to_flush_; - uint32_t frames_written_; - uint32_t append_chunk_index_; - bool is_finalizing_; - - std::shared_ptr connection_pool_; - - void make_buffers_() noexcept; - size_t write_frame_to_chunks_(const uint8_t* buf, size_t buf_size); - bool should_flush_() const; - void compress_buffers_() noexcept; - void flush_(); - [[nodiscard]] virtual bool flush_impl_() = 0; - [[nodiscard]] virtual bool write_array_metadata_() = 0; - virtual bool should_rollover_() const = 0; - void close_sinks_(); - void rollover_(); -}; -} // namespace acquire::sink::zarr diff --git a/src/driver/writers/blosc.compressor.cpp b/src/driver/writers/blosc.compressor.cpp deleted file mode 100644 index 80b6f4eb..00000000 --- a/src/driver/writers/blosc.compressor.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#include "blosc.compressor.hh" - -namespace zarr = acquire::sink::zarr; -using json = nlohmann::json; - -zarr::BloscCompressionParams::BloscCompressionParams() - : clevel{ 1 } - , shuffle{ 1 } -{ -} - -zarr::BloscCompressionParams::BloscCompressionParams( - const std::string& codec_id, - int clevel, - int shuffle) - : codec_id{ codec_id } - , clevel{ clevel } - , shuffle{ shuffle } -{ -} - -void -zarr::to_json(json& j, const zarr::BloscCompressionParams& bcp) -{ - j = json{ { "id", std::string(bcp.id) }, - { "cname", bcp.codec_id }, - { "clevel", bcp.clevel }, - { "shuffle", bcp.shuffle } }; -} - -void -zarr::from_json(const json& j, zarr::BloscCompressionParams& bcp) -{ - j.at("cname").get_to(bcp.codec_id); - j.at("clevel").get_to(bcp.clevel); - j.at("shuffle").get_to(bcp.shuffle); -} \ No newline at end of file diff --git a/src/driver/writers/blosc.compressor.hh b/src/driver/writers/blosc.compressor.hh deleted file mode 100644 index 4b8060b1..00000000 --- a/src/driver/writers/blosc.compressor.hh +++ /dev/null @@ -1,53 +0,0 @@ -#ifndef H_ACQUIRE_ZARR_BLOSC_COMPRESSOR_V0 -#define H_ACQUIRE_ZARR_BLOSC_COMPRESSOR_V0 - -#include "blosc.h" - -#include "nlohmann/json.hpp" - -namespace acquire::sink::zarr { -enum class BloscCodecId : uint8_t -{ - Lz4 = BLOSC_LZ4, - Zstd = BLOSC_ZSTD -}; - -template -constexpr const char* -compression_codec_as_string(); - -template<> -constexpr const char* -compression_codec_as_string() -{ - return "zstd"; -} - -template<> -constexpr const char* -compression_codec_as_string() -{ - return "lz4"; -} - -struct BloscCompressionParams -{ - static constexpr char id[] = "blosc"; - std::string codec_id; - int clevel; - int shuffle; - - BloscCompressionParams(); - BloscCompressionParams(const std::string& codec_id, - int clevel, - int shuffle); -}; - -void -to_json(nlohmann::json&, const BloscCompressionParams&); - -void -from_json(const nlohmann::json&, BloscCompressionParams&); -} - -#endif // H_ACQUIRE_ZARR_BLOSC_COMPRESSOR_V0 diff --git a/src/driver/writers/file.sink.cpp b/src/driver/writers/file.sink.cpp deleted file mode 100644 index 375bde9a..00000000 --- a/src/driver/writers/file.sink.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include "file.sink.hh" -#include "platform.h" -#include "common/utilities.hh" - -#include - -namespace zarr = acquire::sink::zarr; - -zarr::FileSink::FileSink(const std::string& uri) - : file_(new struct file, &file_close) -{ - CHECK(file_); - CHECK(file_create(file_.get(), uri.c_str(), uri.size() + 1)); -} - -bool -zarr::FileSink::write(size_t offset, const uint8_t* buf, size_t bytes_of_buf) -{ - CHECK(buf); - CHECK(bytes_of_buf); - - return file_write(file_.get(), offset, buf, buf + bytes_of_buf); -} diff --git a/src/driver/writers/file.sink.hh b/src/driver/writers/file.sink.hh deleted file mode 100644 index 871df72c..00000000 --- a/src/driver/writers/file.sink.hh +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef H_ACQUIRE_STORAGE_ZARR_WRITERS_FILE_SINK_V0 -#define H_ACQUIRE_STORAGE_ZARR_WRITERS_FILE_SINK_V0 - -#include "sink.hh" -#include "platform.h" - -#include -#include - -namespace acquire::sink::zarr { -struct FileSink : public Sink -{ - public: - FileSink() = delete; - explicit FileSink(const std::string& uri); - - bool write(size_t offset, const uint8_t* buf, size_t bytes_of_buf) override; - - private: - std::unique_ptr file_; -}; -} // namespace acquire::sink::zarr - -#endif // H_ACQUIRE_STORAGE_ZARR_WRITERS_FILE_SINK_V0 diff --git a/src/driver/writers/s3.sink.cpp b/src/driver/writers/s3.sink.cpp deleted file mode 100644 index b746f70c..00000000 --- a/src/driver/writers/s3.sink.cpp +++ /dev/null @@ -1,170 +0,0 @@ -#include "s3.sink.hh" - -#include "common/utilities.hh" -#include "logger.h" - -#include - -namespace zarr = acquire::sink::zarr; - -zarr::S3Sink::S3Sink(std::string_view bucket_name, - std::string_view object_key, - std::shared_ptr connection_pool) - : bucket_name_{ bucket_name } - , object_key_{ object_key } - , connection_pool_{ connection_pool } -{ - CHECK(!bucket_name_.empty()); - CHECK(!object_key_.empty()); - CHECK(connection_pool_); -} - -zarr::S3Sink::~S3Sink() -{ - if (!is_multipart_upload_() && n_bytes_buffered_ > 0) { - if (!put_object_()) { - LOGE("Failed to upload object: %s", object_key_.c_str()); - } - } else if (is_multipart_upload_()) { - if (n_bytes_buffered_ > 0 && !flush_part_()) { - LOGE("Failed to upload part %zu of object %s", - parts_.size() + 1, - object_key_.c_str()); - } - if (!finalize_multipart_upload_()) { - LOGE("Failed to finalize multipart upload of object %s", - object_key_.c_str()); - } - } -} - -bool -zarr::S3Sink::write(size_t _, const uint8_t* data, size_t bytes_of_data) -{ - CHECK(data); - CHECK(bytes_of_data); - - while (bytes_of_data > 0) { - const auto bytes_to_write = - std::min(bytes_of_data, part_buffer_.size() - n_bytes_buffered_); - - if (bytes_to_write) { - std::copy_n( - data, bytes_to_write, part_buffer_.begin() + n_bytes_buffered_); - n_bytes_buffered_ += bytes_to_write; - data += bytes_to_write; - bytes_of_data -= bytes_to_write; - } - - if (n_bytes_buffered_ == part_buffer_.size() && !flush_part_()) { - return false; - } - } - - return true; -} - -bool -zarr::S3Sink::put_object_() -{ - if (n_bytes_buffered_ == 0) { - return false; - } - - auto connection = connection_pool_->get_connection(); - - bool retval = false; - try { - std::string etag = - connection->put_object(bucket_name_, - object_key_, - { part_buffer_.data(), n_bytes_buffered_ }); - EXPECT( - !etag.empty(), "Failed to upload object: %s", object_key_.c_str()); - - retval = true; - } catch (const std::exception& exc) { - LOGE("Error: %s", exc.what()); - } - - // cleanup - connection_pool_->return_connection(std::move(connection)); - n_bytes_buffered_ = 0; - - return retval; -} - -bool -zarr::S3Sink::is_multipart_upload_() const -{ - return !upload_id_.empty() && !parts_.empty(); -} - -std::string -zarr::S3Sink::get_multipart_upload_id_() -{ - if (upload_id_.empty()) { - upload_id_ = - connection_pool_->get_connection()->create_multipart_object( - bucket_name_, object_key_); - } - - return upload_id_; -} - -bool -zarr::S3Sink::flush_part_() -{ - if (n_bytes_buffered_ == 0) { - return false; - } - - auto connection = connection_pool_->get_connection(); - - bool retval = false; - try { - minio::s3::Part part; - part.number = static_cast(parts_.size()) + 1; - - std::span data(part_buffer_.data(), n_bytes_buffered_); - part.etag = - connection->upload_multipart_object_part(bucket_name_, - object_key_, - get_multipart_upload_id_(), - data, - part.number); - EXPECT(!part.etag.empty(), - "Failed to upload part %u of object %s", - part.number, - object_key_.c_str()); - - parts_.push_back(part); - - retval = true; - } catch (const std::exception& exc) { - LOGE("Error: %s", exc.what()); - } - - // cleanup - connection_pool_->return_connection(std::move(connection)); - n_bytes_buffered_ = 0; - - return retval; -} - -bool -zarr::S3Sink::finalize_multipart_upload_() -{ - if (!is_multipart_upload_()) { - return false; - } - - auto connection = connection_pool_->get_connection(); - - bool retval = connection->complete_multipart_object( - bucket_name_, object_key_, upload_id_, parts_); - - connection_pool_->return_connection(std::move(connection)); - - return retval; -} diff --git a/src/driver/writers/s3.sink.hh b/src/driver/writers/s3.sink.hh deleted file mode 100644 index 5b12bce5..00000000 --- a/src/driver/writers/s3.sink.hh +++ /dev/null @@ -1,60 +0,0 @@ -#pragma once - -#include "sink.hh" -#include "platform.h" -#include "common/s3.connection.hh" - -#include - -#include -#include - -namespace acquire::sink::zarr { -class S3Sink final : public Sink -{ - public: - S3Sink() = delete; - S3Sink(std::string_view bucket_name, - std::string_view object_key, - std::shared_ptr connection_pool); - ~S3Sink() override; - - bool write(size_t offset, - const uint8_t* data, - size_t bytes_of_data) override; - - private: - std::string bucket_name_; - std::string object_key_; - - std::shared_ptr connection_pool_; - - // multipart upload - std::array part_buffer_; /// temporary 5MiB buffer for multipart upload - size_t n_bytes_buffered_ = 0; - - std::string upload_id_; - std::list parts_; - - // single-part upload - /// @brief Upload the object to S3. - /// @returns True if the object was successfully uploaded, otherwise false. - [[nodiscard]] bool put_object_(); - - // multipart upload - bool is_multipart_upload_() const; - - /// @brief Get the multipart upload ID, if it exists. Otherwise, create a new - /// multipart upload. - /// @returns The multipart upload ID. - std::string get_multipart_upload_id_(); - - /// @brief Flush the current part to S3. - /// @returns True if the part was successfully flushed, otherwise false. - [[nodiscard]] bool flush_part_(); - /// @brief Finalize the multipart upload. - /// @returns True if a multipart upload was successfully finalized, - /// otherwise false. - [[nodiscard]] bool finalize_multipart_upload_(); -}; -} // namespace acquire::sink::zarr diff --git a/src/driver/writers/sink.creator.cpp b/src/driver/writers/sink.creator.cpp deleted file mode 100644 index 0fd08762..00000000 --- a/src/driver/writers/sink.creator.cpp +++ /dev/null @@ -1,544 +0,0 @@ -#include "sink.creator.hh" -#include "file.sink.hh" -#include "s3.sink.hh" -#include "common/utilities.hh" - -#include -#include - -namespace zarr = acquire::sink::zarr; -namespace common = zarr::common; - -zarr::SinkCreator::SinkCreator( - std::shared_ptr thread_pool_, - std::shared_ptr connection_pool) - : thread_pool_{ thread_pool_ } - , connection_pool_{ connection_pool } -{ -} - -std::unique_ptr -zarr::SinkCreator::make_sink(std::string_view base_uri, std::string_view path) -{ - bool is_s3 = common::is_web_uri(base_uri); - std::string bucket_name; - std::string base_dir; - - if (is_s3) { - common::parse_path_from_uri(base_uri, bucket_name, base_dir); - - // create the bucket if it doesn't already exist - if (!bucket_exists_(bucket_name)) { - return nullptr; - } - } else { - base_dir = base_uri; - if (base_uri.starts_with("file://")) { - base_dir = base_uri.substr(7); - } - - // remove trailing slashes - if (base_uri.ends_with("/") || base_uri.ends_with("\\")) { - base_dir = base_dir.substr(0, base_dir.size() - 1); - } - - // create the parent directory if it doesn't already exist - auto parent_path = - fs::path(base_dir + "/" + std::string(path)).parent_path(); - - if (!fs::is_directory(parent_path) && - !fs::create_directories(parent_path)) { - return nullptr; - } - } - - const std::string full_path = base_dir + "/" + std::string(path); - - std::unique_ptr sink; - if (is_s3) { - sink = - std::make_unique(bucket_name, full_path, connection_pool_); - } else { - sink = std::make_unique(full_path); - } - - return sink; -} - -bool -zarr::SinkCreator::make_data_sinks( - const std::string& base_uri, - const std::vector& dimensions, - const std::function& parts_along_dimension, - std::vector>& part_sinks) -{ - std::queue paths; - - bool is_s3 = common::is_web_uri(base_uri); - std::string bucket_name; - if (is_s3) { - std::string base_dir; - common::parse_path_from_uri(base_uri, bucket_name, base_dir); - - paths.push(base_dir); - } else { - std::string base_dir = base_uri; - - if (base_uri.starts_with("file://")) { - base_dir = base_uri.substr(7); - } - paths.emplace(base_dir); - - if (!make_dirs_(paths)) { - return false; - } - } - - // create directories - for (auto i = dimensions.size() - 2; i >= 1; --i) { - const auto& dim = dimensions.at(i); - const auto n_parts = parts_along_dimension(dim); - CHECK(n_parts); - - auto n_paths = paths.size(); - for (auto j = 0; j < n_paths; ++j) { - const auto path = paths.front(); - paths.pop(); - - for (auto k = 0; k < n_parts; ++k) { - const auto kstr = std::to_string(k); - paths.push(path + (path.empty() ? kstr : "/" + kstr)); - } - } - - if (!is_s3 && !make_dirs_(paths)) { - return false; - } - } - - // create files - { - const auto& dim = dimensions.front(); - const auto n_parts = parts_along_dimension(dim); - CHECK(n_parts); - - auto n_paths = paths.size(); - for (auto i = 0; i < n_paths; ++i) { - const auto path = paths.front(); - paths.pop(); - for (auto j = 0; j < n_parts; ++j) { - paths.push(path + "/" + std::to_string(j)); - } - } - } - - return is_s3 ? make_s3_objects_(bucket_name, paths, part_sinks) - : make_files_(paths, part_sinks); -} - -bool -zarr::SinkCreator::make_metadata_sinks( - acquire::sink::zarr::ZarrVersion version, - const std::string& base_uri, - std::unordered_map>& metadata_sinks) -{ - EXPECT(!base_uri.empty(), "URI must not be empty."); - - std::vector dir_paths, file_paths; - - switch (version) { - case ZarrVersion::V2: - dir_paths.emplace_back("0"); - - file_paths.emplace_back(".zattrs"); - file_paths.emplace_back(".zgroup"); - file_paths.emplace_back("0/.zattrs"); - break; - case ZarrVersion::V3: - dir_paths.emplace_back("meta"); - dir_paths.emplace_back("meta/root"); - - file_paths.emplace_back("zarr.json"); - file_paths.emplace_back("meta/root.group.json"); - break; - default: - throw std::runtime_error("Invalid Zarr version " + - std::to_string(static_cast(version))); - } - - bool is_s3 = common::is_web_uri(base_uri); - std::string bucket_name; - std::string base_dir; - - if (is_s3) { - common::parse_path_from_uri(base_uri, bucket_name, base_dir); - - // create the bucket if it doesn't already exist - if (!bucket_exists_(bucket_name)) { - return false; - } - } else { - base_dir = base_uri; - if (base_uri.starts_with("file://")) { - base_dir = base_uri.substr(7); - } - - // remove trailing slashes - if (base_uri.ends_with("/") || base_uri.ends_with("\\")) { - base_dir = base_dir.substr(0, base_dir.size() - 1); - } - - // create the base directories if they don't already exist - // we create them in serial because - // 1. there are only a few of them; and - // 2. they may be nested - if (!base_dir.empty() && !fs::is_directory(base_dir) && - !fs::create_directories(base_dir)) { - return false; - } - - const std::string prefix = base_dir.empty() ? "" : base_dir + "/"; - for (const auto& dir_path : dir_paths) { - const auto dir = prefix + dir_path; - if (!fs::is_directory(dir) && !fs::create_directories(dir)) { - return false; - } - } - } - - return is_s3 ? make_s3_objects_(bucket_name, file_paths, metadata_sinks) - : make_files_(base_dir, file_paths, metadata_sinks); -} - -bool -zarr::SinkCreator::make_dirs_(std::queue& dir_paths) -{ - if (dir_paths.empty()) { - return true; - } - - std::atomic all_successful = 1; - - const auto n_dirs = dir_paths.size(); - std::latch latch(n_dirs); - - for (auto i = 0; i < n_dirs; ++i) { - const auto dirname = dir_paths.front(); - dir_paths.pop(); - - thread_pool_->push_to_job_queue( - [dirname, &latch, &all_successful](std::string& err) -> bool { - bool success = false; - - try { - if (fs::exists(dirname)) { - EXPECT(fs::is_directory(dirname), - "'%s' exists but is not a directory", - dirname.c_str()); - } else if (all_successful) { - std::error_code ec; - EXPECT(fs::create_directories(dirname, ec), - "%s", - ec.message().c_str()); - } - success = true; - } catch (const std::exception& exc) { - err = "Failed to create directory '" + dirname + - "': " + exc.what(); - } catch (...) { - err = - "Failed to create directory '" + dirname + "': (unknown)."; - } - - latch.count_down(); - all_successful.fetch_and((char)success); - - return success; - }); - - dir_paths.push(dirname); - } - - latch.wait(); - - return (bool)all_successful; -} - -bool -zarr::SinkCreator::make_files_(std::queue& file_paths, - std::vector>& sinks) -{ - if (file_paths.empty()) { - return true; - } - - std::atomic all_successful = 1; - - const auto n_files = file_paths.size(); - sinks.resize(n_files); - std::fill(sinks.begin(), sinks.end(), nullptr); - std::latch latch(n_files); - - for (auto i = 0; i < n_files; ++i) { - const auto filename = file_paths.front(); - file_paths.pop(); - - std::unique_ptr* psink = sinks.data() + i; - - thread_pool_->push_to_job_queue( - [filename, psink, &latch, &all_successful](std::string& err) -> bool { - bool success = false; - - try { - if (all_successful) { - *psink = std::make_unique(filename); - } - success = true; - } catch (const std::exception& exc) { - err = - "Failed to create file '" + filename + "': " + exc.what(); - } catch (...) { - err = "Failed to create file '" + filename + "': (unknown)."; - } - - latch.count_down(); - all_successful.fetch_and((char)success); - - return success; - }); - } - - latch.wait(); - - return (bool)all_successful; -} - -bool -zarr::SinkCreator::make_files_( - const std::string& base_dir, - const std::vector& file_paths, - std::unordered_map>& sinks) -{ - if (file_paths.empty()) { - return true; - } - - std::atomic all_successful = 1; - - const auto n_files = file_paths.size(); - std::latch latch(n_files); - - sinks.clear(); - for (const auto& filename : file_paths) { - sinks[filename] = nullptr; - std::unique_ptr* psink = &sinks[filename]; - - const std::string prefix = base_dir.empty() ? "" : base_dir + "/"; - const auto file_path = prefix + filename; - - thread_pool_->push_to_job_queue( - [filename = file_path, psink, &latch, &all_successful]( - std::string& err) -> bool { - bool success = false; - - try { - if (all_successful) { - *psink = std::make_unique(filename); - } - success = true; - } catch (const std::exception& exc) { - err = - "Failed to create file '" + filename + "': " + exc.what(); - } catch (...) { - err = "Failed to create file '" + filename + "': (unknown)."; - } - - latch.count_down(); - all_successful.fetch_and((char)success); - - return success; - }); - } - - latch.wait(); - - return (bool)all_successful; -} - -bool -zarr::SinkCreator::bucket_exists_(std::string_view bucket_name) -{ - CHECK(!bucket_name.empty()); - EXPECT(connection_pool_, "S3 connection pool not provided."); - - auto conn = connection_pool_->get_connection(); - bool bucket_exists = conn->bucket_exists(bucket_name); - - connection_pool_->return_connection(std::move(conn)); - - return bucket_exists; -} - -bool -zarr::SinkCreator::make_s3_objects_(std::string_view bucket_name, - std::queue& object_keys, - std::vector>& sinks) -{ - if (object_keys.empty()) { - return true; - } - if (bucket_name.empty()) { - LOGE("Bucket name not provided."); - return false; - } - if (!connection_pool_) { - LOGE("S3 connection pool not provided."); - return false; - } - - const auto n_objects = object_keys.size(); - sinks.resize(n_objects); - for (auto i = 0; i < n_objects; ++i) { - sinks[i] = std::make_unique( - bucket_name, object_keys.front(), connection_pool_); - object_keys.pop(); - } - - return true; -} - -bool -zarr::SinkCreator::make_s3_objects_( - std::string_view bucket_name, - std::vector& object_keys, - std::unordered_map>& sinks) -{ - if (object_keys.empty()) { - return true; - } - if (bucket_name.empty()) { - LOGE("Bucket name not provided."); - return false; - } - if (!connection_pool_) { - LOGE("S3 connection pool not provided."); - return false; - } - - sinks.clear(); - for (const auto& key : object_keys) { - sinks[key] = - std::make_unique(bucket_name, key, connection_pool_); - } - - return true; -} - -#ifndef NO_UNIT_TESTS -#ifdef _WIN32 -#define acquire_export __declspec(dllexport) -#else -#define acquire_export -#endif - -extern "C" -{ - acquire_export int unit_test__sink_creator__create_chunk_file_sinks() - { - const fs::path base_dir = fs::temp_directory_path() / "acquire"; - int retval = 0; - - try { - auto thread_pool = std::make_shared( - std::thread::hardware_concurrency(), - [](const std::string& err) { LOGE("Error: %s\n", err.c_str()); }); - zarr::SinkCreator creator{ thread_pool, nullptr }; - - std::vector dims; - dims.emplace_back("x", DimensionType_Space, 10, 2, 0); // 5 chunks - dims.emplace_back("y", DimensionType_Space, 4, 2, 0); // 2 chunks - dims.emplace_back( - "z", DimensionType_Space, 0, 3, 0); // 3 timepoints per chunk - - std::vector> files; - CHECK(creator.make_data_sinks( - base_dir.string(), dims, common::chunks_along_dimension, files)); - - CHECK(files.size() == 5 * 2); - files.clear(); // closes files - - CHECK(fs::is_directory(base_dir)); - for (auto y = 0; y < 2; ++y) { - CHECK(fs::is_directory(base_dir / std::to_string(y))); - for (auto x = 0; x < 5; ++x) { - CHECK(fs::is_regular_file(base_dir / std::to_string(y) / - std::to_string(x))); - } - } - retval = 1; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - - // cleanup - if (fs::exists(base_dir)) { - fs::remove_all(base_dir); - } - return retval; - } - - acquire_export int unit_test__sink_creator__create_shard_file_sinks() - { - const fs::path base_dir = fs::temp_directory_path() / "acquire"; - int retval = 0; - - try { - auto thread_pool = std::make_shared( - std::thread::hardware_concurrency(), - [](const std::string& err) { LOGE("Error: %s", err.c_str()); }); - zarr::SinkCreator creator{ thread_pool, nullptr }; - - std::vector dims; - dims.emplace_back( - "x", DimensionType_Space, 10, 2, 5); // 5 chunks, 1 shard - dims.emplace_back( - "y", DimensionType_Space, 4, 2, 1); // 2 chunks, 2 shards - dims.emplace_back( - "z", DimensionType_Space, 8, 2, 2); // 4 chunks, 2 shards - - std::vector> files; - CHECK(creator.make_data_sinks( - base_dir.string(), dims, common::shards_along_dimension, files)); - - CHECK(files.size() == 2); - files.clear(); // closes files - - CHECK(fs::is_directory(base_dir)); - for (auto y = 0; y < 2; ++y) { - CHECK(fs::is_directory(base_dir / std::to_string(y))); - for (auto x = 0; x < 1; ++x) { - CHECK(fs::is_regular_file(base_dir / std::to_string(y) / - std::to_string(x))); - } - } - - // cleanup - fs::remove_all(base_dir); - - retval = 1; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - - // cleanup - if (fs::exists(base_dir)) { - fs::remove_all(base_dir); - } - return retval; - } -} // extern "C" -#endif // NO_UNIT_TESTS diff --git a/src/driver/writers/sink.creator.hh b/src/driver/writers/sink.creator.hh deleted file mode 100644 index ca2acae0..00000000 --- a/src/driver/writers/sink.creator.hh +++ /dev/null @@ -1,109 +0,0 @@ -#pragma once - -#include "sink.hh" -#include "common/utilities.hh" -#include "common/dimension.hh" -#include "common/thread.pool.hh" -#include "common/s3.connection.hh" - -#include -#include -#include - -namespace acquire::sink::zarr { -class SinkCreator final -{ - public: - SinkCreator() = delete; - SinkCreator(std::shared_ptr thread_pool_, - std::shared_ptr connection_pool); - ~SinkCreator() noexcept = default; - - /// @brief Create a sink from a URI and a path. - /// @param[in] base_uri The base URI for the sink. - /// @param[in] path The path for the sink. - /// @return The sink created, or nullptr if the URI is invalid. - std::unique_ptr make_sink(std::string_view base_uri, - std::string_view path); - - /// @brief Create a collection of data sinks, either chunk or shard. - /// @param[in] base_uri The base URI for the sinks. - /// @param[in] dimensions The dimensions of the data. - /// @param[in] parts_along_dimension Function for computing the number of - /// parts (either chunk or shard) along each dimension. - /// @param[out] part_sinks The sinks created. - /// @return True iff all data sinks were created successfully. - /// @throws std::runtime_error if @p base_uri is not valid, if the number of - /// parts along a dimension cannot be computed, or if, for S3 sinks, - /// the bucket does not exist. - [[nodiscard]] bool make_data_sinks( - const std::string& base_uri, - const std::vector& dimensions, - const std::function& parts_along_dimension, - std::vector>& part_sinks); - - /// @brief Create a collection of metadata sinks for a Zarr dataset. - /// @param[in] version The Zarr version. - /// @param[in] base_uri The base URI for the dataset. - /// @param[out] metadata_sinks The sinks created, keyed by path. - /// @return True iff all metadata sinks were created successfully. - /// @throws std::runtime_error if @p base_uri is not valid, or if, for S3 - /// sinks, the bucket does not exist. - [[nodiscard]] bool make_metadata_sinks( - ZarrVersion version, - const std::string& base_uri, - std::unordered_map>& metadata_sinks); - - private: - std::shared_ptr thread_pool_; - std::shared_ptr connection_pool_; // could be null - - /// @brief Parallel create a collection of directories. - /// @param[in] dir_paths The directories to create. - /// @return True iff all directories were created successfully. - [[nodiscard]] bool make_dirs_(std::queue& dir_paths); - - /// @brief Parallel create a collection of files. - /// @param[in,out] file_paths The files to create. Unlike `make_dirs_`, - /// this function drains the queue. - /// @param[out] files The files created. - /// @return True iff all files were created successfully. - [[nodiscard]] bool make_files_(std::queue& file_paths, - std::vector>& sinks); - - /// @brief Parallel create a collection of files, keyed by path. - /// @param[in] base_dir The base directory for the files. - /// @param[in] file_paths Paths to the files to create, relative to @p base_dir. - /// @param[out] sinks The sinks created, keyed by path. - /// @return True iff all files were created successfully. - [[nodiscard]] bool make_files_( - const std::string& base_dir, - const std::vector& file_paths, - std::unordered_map>& sinks); - - /// @brief Check whether an S3 bucket exists. - /// @param[in] bucket_name The name of the bucket to check. - /// @return True iff the bucket exists. - bool bucket_exists_(std::string_view bucket_name); - - /// @brief Create a collection of S3 objects. - /// @param[in] bucket_name The name of the bucket. - /// @param[in,out] object_keys The keys of the objects to create. - /// @param[out] sinks The sinks created. - /// @return True iff all S3 objects were created successfully. - [[nodiscard]] bool make_s3_objects_( - std::string_view bucket_name, - std::queue& object_keys, - std::vector>& sinks); - - /// @brief Create a collection of S3 objects, keyed by object key. - /// @param[in] bucket_name The name of the bucket. - /// @param[in] object_keys The keys of the objects to create. - /// @param[out] sinks The sinks created, keyed by object key. - /// @return True iff all S3 objects were created successfully. - [[nodiscard]] bool make_s3_objects_( - std::string_view bucket_name, - std::vector& object_keys, - std::unordered_map>& sinks); -}; -} // namespace acquire::sink::zarr diff --git a/src/driver/writers/sink.hh b/src/driver/writers/sink.hh deleted file mode 100644 index 96b61d37..00000000 --- a/src/driver/writers/sink.hh +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef H_ACQUIRE_STORAGE_ZARR_WRITERS_SINK_V0 -#define H_ACQUIRE_STORAGE_ZARR_WRITERS_SINK_V0 - -#include // uint8_t -#include // size_t - -namespace acquire::sink::zarr { -struct Sink -{ - virtual ~Sink() noexcept = default; - - /// @brief Write data to the sink. - /// @param offset The offset in the sink to write to. - /// @param buf The buffer to write to the sink. - /// @param bytes_of_buf The number of bytes to write from @p buf. - /// @return True if the write was successful, false otherwise. - /// @throws std::runtime_error if @p buf is nullptr, @p bytes_of_buf is 0, - /// or the write fails. - [[nodiscard]] virtual bool write(size_t offset, - const uint8_t* buf, - size_t bytes_of_buf) = 0; -}; -} // namespace acquire::sink::zarr -#endif // H_ACQUIRE_STORAGE_ZARR_WRITERS_SINK_V0 diff --git a/src/driver/writers/zarrv2.array.writer.cpp b/src/driver/writers/zarrv2.array.writer.cpp deleted file mode 100644 index 0cf26433..00000000 --- a/src/driver/writers/zarrv2.array.writer.cpp +++ /dev/null @@ -1,554 +0,0 @@ -#include "zarrv2.array.writer.hh" -#include "sink.creator.hh" -#include "zarr.hh" - -#include -#include -#include - -namespace zarr = acquire::sink::zarr; - -namespace { - -std::string -sample_type_to_dtype(SampleType t) - -{ - const std::string dtype_prefix = - std::endian::native == std::endian::big ? ">" : "<"; - - switch (t) { - case SampleType_u8: - return dtype_prefix + "u1"; - case SampleType_u10: - case SampleType_u12: - case SampleType_u14: - case SampleType_u16: - return dtype_prefix + "u2"; - case SampleType_i8: - return dtype_prefix + "i1"; - case SampleType_i16: - return dtype_prefix + "i2"; - case SampleType_f32: - return dtype_prefix + "f4"; - default: - throw std::runtime_error("Invalid SampleType: " + - std::to_string(static_cast(t))); - } -} -} // end ::{anonymous} namespace - -zarr::ZarrV2ArrayWriter::ZarrV2ArrayWriter( - const ArrayWriterConfig& config, - std::shared_ptr thread_pool, - std::shared_ptr connection_pool) - : ArrayWriter(config, thread_pool, connection_pool) -{ - data_root_ = - config_.dataset_root + "/" + std::to_string(config_.level_of_detail); - meta_root_ = data_root_; -} - -bool -zarr::ZarrV2ArrayWriter::flush_impl_() -{ - // create chunk files - CHECK(data_sinks_.empty()); - const std::string data_root = - data_root_ + "/" + std::to_string(append_chunk_index_); - - { - SinkCreator creator(thread_pool_, connection_pool_); - if (!creator.make_data_sinks(data_root, - config_.dimensions, - common::chunks_along_dimension, - data_sinks_)) { - return false; - } - } - - CHECK(data_sinks_.size() == chunk_buffers_.size()); - - std::latch latch(chunk_buffers_.size()); - { - std::scoped_lock lock(buffers_mutex_); - for (auto i = 0; i < data_sinks_.size(); ++i) { - auto& chunk = chunk_buffers_.at(i); - thread_pool_->push_to_job_queue( - std::move([&sink = data_sinks_.at(i), - data = chunk.data(), - size = chunk.size(), - &latch](std::string& err) -> bool { - bool success = false; - try { - CHECK(sink->write(0, data, size)); - success = true; - } catch (const std::exception& exc) { - err = "Failed to write chunk: " + std::string(exc.what()); - } catch (...) { - err = "Failed to write chunk: (unknown)"; - } - - latch.count_down(); - return success; - })); - } - } - - // wait for all threads to finish - latch.wait(); - - return true; -} - -bool -zarr::ZarrV2ArrayWriter::write_array_metadata_() -{ - if (!metadata_sink_) { - const std::string metadata_path = ".zarray"; - SinkCreator creator(thread_pool_, connection_pool_); - if (!(metadata_sink_ = creator.make_sink(meta_root_, metadata_path))) { - LOGE("Failed to create metadata sink: %s/%s", - meta_root_.c_str(), - metadata_path.c_str()); - return false; - } - } - - namespace fs = std::filesystem; - using json = nlohmann::json; - - const auto& image_shape = config_.image_shape; - - std::vector array_shape, chunk_shape; - - size_t append_size = frames_written_; - for (auto dim = config_.dimensions.begin() + 2; - dim < config_.dimensions.end() - 1; - ++dim) { - CHECK(dim->array_size_px); - append_size = (append_size + dim->array_size_px - 1) / dim->array_size_px; - } - array_shape.push_back(append_size); - - chunk_shape.push_back(config_.dimensions.back().chunk_size_px); - for (auto dim = config_.dimensions.rbegin() + 1; - dim != config_.dimensions.rend(); - ++dim) { - array_shape.push_back(dim->array_size_px); - chunk_shape.push_back(dim->chunk_size_px); - } - - json metadata; - metadata["zarr_format"] = 2; - metadata["shape"] = array_shape; - metadata["chunks"] = chunk_shape; - metadata["dtype"] = sample_type_to_dtype(image_shape.type); - metadata["fill_value"] = 0; - metadata["order"] = "C"; - metadata["filters"] = nullptr; - metadata["dimension_separator"] = "/"; - - if (config_.compression_params) { - metadata["compressor"] = *config_.compression_params; - } else { - metadata["compressor"] = nullptr; - } - - const std::string metadata_str = metadata.dump(4); - const auto* metadata_bytes = (const uint8_t*)metadata_str.c_str(); - - return metadata_sink_->write(0, metadata_bytes, metadata_str.size()); -} - -bool -zarr::ZarrV2ArrayWriter::should_rollover_() const -{ - return true; -} - -#ifndef NO_UNIT_TESTS -#ifdef _WIN32 -#define acquire_export __declspec(dllexport) -#else -#define acquire_export -#endif - -namespace common = zarr::common; - -extern "C" -{ - acquire_export int unit_test__zarrv2_writer__write_even() - { - int retval = 0; - const fs::path base_dir = fs::temp_directory_path() / "acquire"; - - const unsigned int array_width = 64, array_height = 48, - array_planes = 6, array_channels = 8, - array_timepoints = 10; - const unsigned int n_frames = - array_planes * array_channels * array_timepoints; - - const unsigned int chunk_width = 16, chunk_height = 16, - chunk_planes = 2, chunk_channels = 4, - chunk_timepoints = 5; - - const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks - const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks - - const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks - const unsigned int chunks_in_c = - (array_channels + chunk_channels - 1) / chunk_channels; // 2 chunks - const unsigned int chunks_in_t = - (array_timepoints + chunk_timepoints - 1) / - chunk_timepoints; // 2 chunks - - const ImageShape shape - { - .dims = { - .width = array_width, - .height = array_height, - }, - .strides = { - .width = 1, - .height = array_width, - .planes = array_width * array_height, - }, - .type = SampleType_u16, - }; - const unsigned int nbytes_px = bytes_of_type(shape.type); - - try { - auto thread_pool = std::make_shared( - std::thread::hardware_concurrency(), - [](const std::string& err) { LOGE("Error: %s\n", err.c_str()); }); - - std::vector dims; - dims.emplace_back( - "x", DimensionType_Space, array_width, chunk_width, 0); - dims.emplace_back( - "y", DimensionType_Space, array_height, chunk_height, 0); - dims.emplace_back( - "z", DimensionType_Space, array_planes, chunk_planes, 0); - dims.emplace_back( - "c", DimensionType_Channel, array_channels, chunk_channels, 0); - dims.emplace_back( - "t", DimensionType_Time, array_timepoints, chunk_timepoints, 0); - - zarr::ArrayWriterConfig config = { - .image_shape = shape, - .dimensions = dims, - .level_of_detail = 0, - .dataset_root = base_dir.string(), - .compression_params = std::nullopt, - }; - - zarr::ZarrV2ArrayWriter writer(config, thread_pool, nullptr); - - const size_t frame_size = array_width * array_height * nbytes_px; - std::vector data(frame_size, 0); - - for (auto i = 0; i < n_frames; ++i) { // 2 time points - CHECK(writer.write(data.data(), frame_size)); - } - writer.finalize(); - - const auto expected_file_size = chunk_width * chunk_height * - chunk_planes * chunk_channels * - chunk_timepoints * nbytes_px; - - const fs::path data_root = - base_dir / std::to_string(config.level_of_detail); - CHECK(fs::is_directory(data_root)); - for (auto t = 0; t < chunks_in_t; ++t) { - const auto t_dir = data_root / std::to_string(t); - CHECK(fs::is_directory(t_dir)); - - for (auto c = 0; c < chunks_in_c; ++c) { - const auto c_dir = t_dir / std::to_string(c); - CHECK(fs::is_directory(c_dir)); - - for (auto z = 0; z < chunks_in_z; ++z) { - const auto z_dir = c_dir / std::to_string(z); - CHECK(fs::is_directory(z_dir)); - - for (auto y = 0; y < chunks_in_y; ++y) { - const auto y_dir = z_dir / std::to_string(y); - CHECK(fs::is_directory(y_dir)); - - for (auto x = 0; x < chunks_in_x; ++x) { - const auto x_file = y_dir / std::to_string(x); - CHECK(fs::is_regular_file(x_file)); - const auto file_size = fs::file_size(x_file); - CHECK(file_size == expected_file_size); - } - - CHECK(!fs::is_regular_file( - y_dir / std::to_string(chunks_in_x))); - } - - CHECK(!fs::is_directory(z_dir / - std::to_string(chunks_in_y))); - } - - CHECK( - !fs::is_directory(c_dir / std::to_string(chunks_in_z))); - } - - CHECK(!fs::is_directory(t_dir / std::to_string(chunks_in_c))); - } - - CHECK(!fs::is_directory(base_dir / std::to_string(chunks_in_t))); - - retval = 1; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - - // cleanup - if (fs::exists(base_dir)) { - fs::remove_all(base_dir); - } - return retval; - } - - acquire_export int unit_test__zarrv2_writer__write_ragged_append_dim() - { - int retval = 0; - const fs::path base_dir = fs::temp_directory_path() / "acquire"; - - const unsigned int array_width = 64, array_height = 48, - array_planes = 5; - const unsigned int n_frames = array_planes; - - const unsigned int chunk_width = 16, chunk_height = 16, - chunk_planes = 2; - - const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks - const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks - - const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks, ragged - - const ImageShape shape - { - .dims = { - .width = array_width, - .height = array_height, - }, - .strides = { - .width = 1, - .height = array_width, - .planes = array_width * array_height, - }, - .type = SampleType_u8, - }; - const unsigned int nbytes_px = bytes_of_type(shape.type); - - try { - auto thread_pool = std::make_shared( - std::thread::hardware_concurrency(), - [](const std::string& err) { LOGE("Error: %s", err.c_str()); }); - - std::vector dims; - dims.emplace_back( - "x", DimensionType_Space, array_width, chunk_width, 0); - dims.emplace_back( - "y", DimensionType_Space, array_height, chunk_height, 0); - dims.emplace_back( - "z", DimensionType_Space, array_planes, chunk_planes, 0); - - zarr::ArrayWriterConfig config = { - .image_shape = shape, - .dimensions = dims, - .level_of_detail = 1, - .dataset_root = base_dir.string(), - .compression_params = std::nullopt, - }; - - zarr::ZarrV2ArrayWriter writer( - config, thread_pool, std::shared_ptr()); - - const size_t frame_size = array_width * array_height * nbytes_px; - std::vector data(frame_size, 0); - - for (auto i = 0; i < n_frames; ++i) { - CHECK(writer.write(data.data(), frame_size) == frame_size); - } - writer.finalize(); - - const auto expected_file_size = - chunk_width * chunk_height * chunk_planes; - - const fs::path data_root = - base_dir / std::to_string(config.level_of_detail); - CHECK(fs::is_directory(data_root)); - for (auto z = 0; z < chunks_in_z; ++z) { - const auto z_dir = data_root / std::to_string(z); - CHECK(fs::is_directory(z_dir)); - - for (auto y = 0; y < chunks_in_y; ++y) { - const auto y_dir = z_dir / std::to_string(y); - CHECK(fs::is_directory(y_dir)); - - for (auto x = 0; x < chunks_in_x; ++x) { - const auto x_file = y_dir / std::to_string(x); - CHECK(fs::is_regular_file(x_file)); - const auto file_size = fs::file_size(x_file); - CHECK(file_size == expected_file_size); - } - - CHECK(!fs::is_regular_file(y_dir / - std::to_string(chunks_in_x))); - } - - CHECK(!fs::is_directory(z_dir / std::to_string(chunks_in_y))); - } - - CHECK(!fs::is_directory(base_dir / std::to_string(chunks_in_z))); - - retval = 1; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - - // cleanup - if (fs::exists(base_dir)) { - fs::remove_all(base_dir); - } - return retval; - } - - acquire_export int unit_test__zarrv2_writer__write_ragged_internal_dim() - { - int retval = 0; - const fs::path base_dir = fs::temp_directory_path() / "acquire"; - - const unsigned int array_width = 64, array_height = 48, - array_planes = 5, array_timepoints = 5; - const unsigned int n_frames = array_planes * array_timepoints; - - const unsigned int chunk_width = 16, chunk_height = 16, - chunk_planes = 2, chunk_timepoints = 5; - - const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks - const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks - - const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks, ragged - const unsigned int chunks_in_t = - (array_timepoints + chunk_timepoints - 1) / - chunk_timepoints; // 1 chunk - - const ImageShape shape - { - .dims = { - .width = array_width, - .height = array_height, - }, - .strides = { - .width = 1, - .height = array_width, - .planes = array_width * array_height, - }, - .type = SampleType_u8, - }; - const unsigned int nbytes_px = bytes_of_type(shape.type); - - try { - auto thread_pool = std::make_shared( - std::thread::hardware_concurrency(), - [](const std::string& err) { LOGE("Error: %s", err.c_str()); }); - - std::vector dims; - dims.emplace_back( - "x", DimensionType_Space, array_width, chunk_width, 0); - dims.emplace_back( - "y", DimensionType_Space, array_height, chunk_height, 0); - dims.emplace_back( - "z", DimensionType_Space, array_planes, chunk_planes, 0); - dims.emplace_back( - "t", DimensionType_Time, array_timepoints, chunk_timepoints, 0); - - zarr::ArrayWriterConfig config = { - .image_shape = shape, - .dimensions = dims, - .level_of_detail = 2, - .dataset_root = base_dir.string(), - .compression_params = std::nullopt, - }; - - zarr::ZarrV2ArrayWriter writer( - config, thread_pool, std::shared_ptr()); - - const size_t frame_size = array_width * array_height * nbytes_px; - std::vector data(frame_size, 0); - - for (auto i = 0; i < n_frames; ++i) { - CHECK(writer.write(data.data(), frame_size) == frame_size); - } - writer.finalize(); - - const auto expected_file_size = - chunk_width * chunk_height * chunk_planes * chunk_timepoints; - - const fs::path data_root = - base_dir / std::to_string(config.level_of_detail); - CHECK(fs::is_directory(data_root)); - for (auto t = 0; t < chunks_in_t; ++t) { - const auto t_dir = data_root / std::to_string(t); - CHECK(fs::is_directory(t_dir)); - - for (auto z = 0; z < chunks_in_z; ++z) { - const auto z_dir = t_dir / std::to_string(z); - CHECK(fs::is_directory(z_dir)); - - for (auto y = 0; y < chunks_in_y; ++y) { - const auto y_dir = z_dir / std::to_string(y); - CHECK(fs::is_directory(y_dir)); - - for (auto x = 0; x < chunks_in_x; ++x) { - const auto x_file = y_dir / std::to_string(x); - CHECK(fs::is_regular_file(x_file)); - const auto file_size = fs::file_size(x_file); - CHECK(file_size == expected_file_size); - } - - CHECK(!fs::is_regular_file( - y_dir / std::to_string(chunks_in_x))); - } - - CHECK( - !fs::is_directory(z_dir / std::to_string(chunks_in_y))); - } - - CHECK(!fs::is_directory(t_dir / std::to_string(chunks_in_z))); - } - CHECK(!fs::is_directory(base_dir / std::to_string(chunks_in_t))); - - retval = 1; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - - // cleanup - if (fs::exists(base_dir)) { - fs::remove_all(base_dir); - } - return retval; - } -} -#endif diff --git a/src/driver/writers/zarrv2.array.writer.hh b/src/driver/writers/zarrv2.array.writer.hh deleted file mode 100644 index aac94bfb..00000000 --- a/src/driver/writers/zarrv2.array.writer.hh +++ /dev/null @@ -1,34 +0,0 @@ -#pragma once - -#include "array.writer.hh" - -#include "platform.h" -#include "device/props/components.h" - -#include -#include -#include -#include -#include -#include - -namespace fs = std::filesystem; - -namespace acquire::sink::zarr { -struct ZarrV2ArrayWriter final : public ArrayWriter -{ - public: - ZarrV2ArrayWriter() = delete; - ZarrV2ArrayWriter( - const ArrayWriterConfig& config, - std::shared_ptr thread_pool, - std::shared_ptr connection_pool); - - ~ZarrV2ArrayWriter() override = default; - - private: - bool flush_impl_() override; - bool write_array_metadata_() override; - bool should_rollover_() const override; -}; -} // namespace acquire::sink::zarr diff --git a/src/driver/writers/zarrv3.array.writer.cpp b/src/driver/writers/zarrv3.array.writer.cpp deleted file mode 100644 index ba44007a..00000000 --- a/src/driver/writers/zarrv3.array.writer.cpp +++ /dev/null @@ -1,733 +0,0 @@ -#include "zarrv3.array.writer.hh" -#include "sink.creator.hh" -#include "zarr.hh" - -#include -#include - -namespace zarr = acquire::sink::zarr; - -namespace { -std::string -sample_type_to_dtype(SampleType t) - -{ - switch (t) { - case SampleType_u8: - return "uint8"; - case SampleType_u10: - case SampleType_u12: - case SampleType_u14: - case SampleType_u16: - return "uint16"; - case SampleType_i8: - return "int8"; - case SampleType_i16: - return "int16"; - case SampleType_f32: - return "float32"; - default: - throw std::runtime_error("Invalid SampleType: " + - std::to_string(static_cast(t))); - } -} -} // end ::{anonymous} namespace - -zarr::ZarrV3ArrayWriter::ZarrV3ArrayWriter( - const ArrayWriterConfig& array_spec, - std::shared_ptr thread_pool, - std::shared_ptr connection_pool) - : ArrayWriter(array_spec, thread_pool, connection_pool) - , shard_file_offsets_(common::number_of_shards(array_spec.dimensions), 0) - , shard_tables_{ common::number_of_shards(array_spec.dimensions) } -{ - const auto chunks_per_shard = - common::chunks_per_shard(array_spec.dimensions); - - for (auto& table : shard_tables_) { - table.resize(2 * chunks_per_shard); - std::fill_n( - table.begin(), table.size(), std::numeric_limits::max()); - } - - data_root_ = config_.dataset_root + "/data/root/" + - std::to_string(config_.level_of_detail); - meta_root_ = config_.dataset_root + "/meta/root"; -} - -bool -zarr::ZarrV3ArrayWriter::flush_impl_() -{ - // create shard files if they don't exist - const std::string data_root = - data_root_ + "/c" + std::to_string(append_chunk_index_); - - { - SinkCreator creator(thread_pool_, connection_pool_); - if (data_sinks_.empty() && - !creator.make_data_sinks(data_root, - config_.dimensions, - common::shards_along_dimension, - data_sinks_)) { - return false; - } - } - - const auto n_shards = common::number_of_shards(config_.dimensions); - CHECK(data_sinks_.size() == n_shards); - - // get shard indices for each chunk - std::vector> chunk_in_shards(n_shards); - for (auto i = 0; i < chunk_buffers_.size(); ++i) { - const auto index = common::shard_index_for_chunk(i, config_.dimensions); - chunk_in_shards.at(index).push_back(i); - } - - // write out chunks to shards - bool write_table = is_finalizing_ || should_rollover_(); - std::latch latch(n_shards); - for (auto i = 0; i < n_shards; ++i) { - const auto& chunks = chunk_in_shards.at(i); - auto& chunk_table = shard_tables_.at(i); - size_t* file_offset = &shard_file_offsets_.at(i); - - thread_pool_->push_to_job_queue([&sink = data_sinks_.at(i), - &chunks, - &chunk_table, - file_offset, - write_table, - &latch, - this](std::string& err) mutable { - bool success = false; - - try { - for (const auto& chunk_idx : chunks) { - auto& chunk = chunk_buffers_.at(chunk_idx); - - success = - sink->write(*file_offset, chunk.data(), chunk.size()); - if (!success) { - break; - } - - const auto internal_idx = common::shard_internal_index( - chunk_idx, config_.dimensions); - chunk_table.at(2 * internal_idx) = *file_offset; - chunk_table.at(2 * internal_idx + 1) = chunk.size(); - - *file_offset += chunk.size(); - } - - if (success && write_table) { - const auto* table = - reinterpret_cast(chunk_table.data()); - success = - sink->write(*file_offset, - table, - chunk_table.size() * sizeof(uint64_t)); - } - } catch (const std::exception& exc) { - err = "Failed to write chunk: " + std::string(exc.what()); - } catch (...) { - err = "Failed to write chunk: (unknown)"; - } - - latch.count_down(); - return success; - }); - } - - // wait for all threads to finish - latch.wait(); - - // reset shard tables and file offsets - if (write_table) { - for (auto& table : shard_tables_) { - std::fill_n(table.begin(), - table.size(), - std::numeric_limits::max()); - } - - std::fill_n(shard_file_offsets_.begin(), shard_file_offsets_.size(), 0); - } - - return true; -} - -bool -zarr::ZarrV3ArrayWriter::write_array_metadata_() -{ - if (!metadata_sink_) { - const std::string metadata_path = - std::to_string(config_.level_of_detail) + ".array.json"; - SinkCreator creator(thread_pool_, connection_pool_); - if (!(metadata_sink_ = creator.make_sink(meta_root_, metadata_path))) { - LOGE("Failed to create metadata sink: %s/%s", - meta_root_.c_str(), - metadata_path.c_str()); - return false; - } - } - - namespace fs = std::filesystem; - using json = nlohmann::json; - - const auto& image_shape = config_.image_shape; - - std::vector array_shape, chunk_shape, shard_shape; - - size_t append_size = frames_written_; - for (auto dim = config_.dimensions.begin() + 2; - dim < config_.dimensions.end() - 1; - ++dim) { - CHECK(dim->array_size_px); - append_size = (append_size + dim->array_size_px - 1) / dim->array_size_px; - } - array_shape.push_back(append_size); - - chunk_shape.push_back(config_.dimensions.back().chunk_size_px); - shard_shape.push_back(config_.dimensions.back().shard_size_chunks); - for (auto dim = config_.dimensions.rbegin() + 1; - dim != config_.dimensions.rend(); - ++dim) { - array_shape.push_back(dim->array_size_px); - chunk_shape.push_back(dim->chunk_size_px); - shard_shape.push_back(dim->shard_size_chunks); - } - - json metadata; - metadata["attributes"] = json::object(); - metadata["chunk_grid"] = json::object({ - { "chunk_shape", chunk_shape }, - { "separator", "/" }, - { "type", "regular" }, - }); - - metadata["chunk_memory_layout"] = "C"; - metadata["data_type"] = sample_type_to_dtype(image_shape.type); - metadata["extensions"] = json::array(); - metadata["fill_value"] = 0; - metadata["shape"] = array_shape; - - if (config_.compression_params) { - const auto params = *config_.compression_params; - metadata["compressor"] = json::object({ - { "codec", "https://purl.org/zarr/spec/codec/blosc/1.0" }, - { "configuration", - json::object({ - { "blocksize", 0 }, - { "clevel", params.clevel }, - { "cname", params.codec_id }, - { "shuffle", params.shuffle }, - }) }, - }); - } - - // sharding storage transformer - // TODO (aliddell): - // https://github.com/zarr-developers/zarr-python/issues/877 - metadata["storage_transformers"] = json::array(); - metadata["storage_transformers"][0] = json::object({ - { "type", "indexed" }, - { "extension", - "https://purl.org/zarr/spec/storage_transformers/sharding/1.0" }, - { "configuration", - json::object({ - { "chunks_per_shard", shard_shape }, - }) }, - }); - - const std::string metadata_str = metadata.dump(4); - const auto* metadata_bytes = (const uint8_t*)metadata_str.c_str(); - - return metadata_sink_->write(0, metadata_bytes, metadata_str.size()); -} - -bool -zarr::ZarrV3ArrayWriter::should_rollover_() const -{ - const auto& dims = config_.dimensions; - size_t frames_before_flush = - dims.back().chunk_size_px * dims.back().shard_size_chunks; - for (auto i = 2; i < dims.size() - 1; ++i) { - frames_before_flush *= dims[i].array_size_px; - } - - CHECK(frames_before_flush > 0); - return frames_written_ % frames_before_flush == 0; -} - -#ifndef NO_UNIT_TESTS -#ifdef _WIN32 -#define acquire_export __declspec(dllexport) -#else -#define acquire_export -#endif - -namespace common = zarr::common; - -extern "C" -{ - acquire_export int unit_test__zarrv3_writer__write_even() - { - int retval = 0; - const fs::path base_dir = fs::temp_directory_path() / "acquire"; - - const unsigned int array_width = 64, array_height = 48, - array_planes = 6, array_channels = 8, - array_timepoints = 10; - const unsigned int n_frames = - array_planes * array_channels * array_timepoints; - - const unsigned int chunk_width = 16, chunk_height = 16, - chunk_planes = 2, chunk_channels = 4, - chunk_timepoints = 5; - - const unsigned int shard_width = 2, shard_height = 1, shard_planes = 1, - shard_channels = 2, shard_timepoints = 2; - const unsigned int chunks_per_shard = shard_width * shard_height * - shard_planes * shard_channels * - shard_timepoints; - - const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks - const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks - const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks - const unsigned int chunks_in_c = - (array_channels + chunk_channels - 1) / chunk_channels; // 2 chunks - const unsigned int chunks_in_t = - (array_timepoints + chunk_timepoints - 1) / chunk_timepoints; - - const unsigned int shards_in_x = - (chunks_in_x + shard_width - 1) / shard_width; // 2 shards - const unsigned int shards_in_y = - (chunks_in_y + shard_height - 1) / shard_height; // 3 shards - const unsigned int shards_in_z = - (chunks_in_z + shard_planes - 1) / shard_planes; // 3 shards - const unsigned int shards_in_c = - (chunks_in_c + shard_channels - 1) / shard_channels; // 1 shard - const unsigned int shards_in_t = - (chunks_in_t + shard_timepoints - 1) / shard_timepoints; // 1 shard - - const ImageShape shape - { - .dims = { - .width = array_width, - .height = array_height, - }, - .strides = { - .width = 1, - .height = array_width, - .planes = array_width * array_height, - }, - .type = SampleType_u16, - }; - const unsigned int nbytes_px = bytes_of_type(shape.type); - - try { - auto thread_pool = std::make_shared( - std::thread::hardware_concurrency(), - [](const std::string& err) { LOGE("Error: %s", err.c_str()); }); - - std::vector dims; - dims.emplace_back( - "x", DimensionType_Space, array_width, chunk_width, shard_width); - dims.emplace_back("y", - DimensionType_Space, - array_height, - chunk_height, - shard_height); - dims.emplace_back("z", - DimensionType_Space, - array_planes, - chunk_planes, - shard_planes); - dims.emplace_back("c", - DimensionType_Channel, - array_channels, - chunk_channels, - shard_channels); - dims.emplace_back("t", - DimensionType_Time, - array_timepoints, - chunk_timepoints, - shard_timepoints); - - zarr::ArrayWriterConfig config = { - .image_shape = shape, - .dimensions = dims, - .level_of_detail = 3, - .dataset_root = base_dir.string(), - .compression_params = std::nullopt, - }; - - zarr::ZarrV3ArrayWriter writer( - config, thread_pool, std::shared_ptr()); - - const size_t frame_size = array_width * array_height * nbytes_px; - std::vector data(frame_size, 0); - - for (auto i = 0; i < n_frames; ++i) { - CHECK(writer.write(data.data(), frame_size)); - } - writer.finalize(); - - const auto chunk_size = chunk_width * chunk_height * chunk_planes * - chunk_channels * chunk_timepoints * - nbytes_px; - const auto index_size = chunks_per_shard * - sizeof(uint64_t) * // indices are 64 bits - 2; // 2 indices per chunk - const auto expected_file_size = shard_width * shard_height * - shard_planes * shard_channels * - shard_timepoints * chunk_size + - index_size; - - const fs::path data_root = - base_dir / "data/root" / std::to_string(config.level_of_detail); - CHECK(fs::is_directory(data_root)); - for (auto t = 0; t < shards_in_t; ++t) { - const auto t_dir = data_root / ("c" + std::to_string(t)); - CHECK(fs::is_directory(t_dir)); - - for (auto c = 0; c < shards_in_c; ++c) { - const auto c_dir = t_dir / std::to_string(c); - CHECK(fs::is_directory(c_dir)); - - for (auto z = 0; z < shards_in_z; ++z) { - const auto z_dir = c_dir / std::to_string(z); - CHECK(fs::is_directory(z_dir)); - - for (auto y = 0; y < shards_in_y; ++y) { - const auto y_dir = z_dir / std::to_string(y); - CHECK(fs::is_directory(y_dir)); - - for (auto x = 0; x < shards_in_x; ++x) { - const auto x_file = y_dir / std::to_string(x); - CHECK(fs::is_regular_file(x_file)); - const auto file_size = fs::file_size(x_file); - CHECK(file_size == expected_file_size); - } - - CHECK(!fs::is_regular_file( - y_dir / std::to_string(shards_in_x))); - } - - CHECK(!fs::is_directory(z_dir / - std::to_string(shards_in_y))); - } - - CHECK( - !fs::is_directory(c_dir / std::to_string(shards_in_z))); - } - - CHECK(!fs::is_directory(t_dir / std::to_string(shards_in_c))); - } - - CHECK(!fs::is_directory(base_dir / - ("c" + std::to_string(shards_in_t)))); - - retval = 1; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - - // cleanup - if (fs::exists(base_dir)) { - fs::remove_all(base_dir); - } - return retval; - } - - acquire_export int unit_test__zarrv3_writer__write_ragged_append_dim() - { - int retval = 0; - const fs::path base_dir = fs::temp_directory_path() / "acquire"; - - const unsigned int array_width = 64, array_height = 48, - array_planes = 5; - const unsigned int n_frames = array_planes; - - const unsigned int chunk_width = 16, chunk_height = 16, - chunk_planes = 2; - - const unsigned int shard_width = 2, shard_height = 1, shard_planes = 1; - const unsigned int chunks_per_shard = - shard_width * shard_height * shard_planes; - - const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks - const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks - const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks - - const unsigned int shards_in_x = - (chunks_in_x + shard_width - 1) / shard_width; // 2 shards - const unsigned int shards_in_y = - (chunks_in_y + shard_height - 1) / shard_height; // 3 shards - const unsigned int shards_in_z = - (chunks_in_z + shard_planes - 1) / shard_planes; // 3 shards - - const ImageShape shape - { - .dims = { - .width = array_width, - .height = array_height, - }, - .strides = { - .width = 1, - .height = array_width, - .planes = array_width * array_height, - }, - .type = SampleType_i8, - }; - const unsigned int nbytes_px = bytes_of_type(shape.type); - - try { - auto thread_pool = std::make_shared( - std::thread::hardware_concurrency(), - [](const std::string& err) { LOGE("Error: %s", err.c_str()); }); - - std::vector dims; - dims.emplace_back( - "x", DimensionType_Space, array_width, chunk_width, shard_width); - dims.emplace_back("y", - DimensionType_Space, - array_height, - chunk_height, - shard_height); - dims.emplace_back("z", - DimensionType_Space, - array_planes, - chunk_planes, - shard_planes); - - zarr::ArrayWriterConfig config = { - .image_shape = shape, - .dimensions = dims, - .level_of_detail = 4, - .dataset_root = base_dir.string(), - .compression_params = std::nullopt, - }; - - zarr::ZarrV3ArrayWriter writer( - config, thread_pool, std::shared_ptr()); - - const size_t frame_size = array_width * array_height * nbytes_px; - std::vector data(frame_size, 0); - - for (auto i = 0; i < n_frames; ++i) { - CHECK(writer.write(data.data(), frame_size) == frame_size); - } - writer.finalize(); - - const auto chunk_size = - chunk_width * chunk_height * chunk_planes * nbytes_px; - const auto index_size = chunks_per_shard * - sizeof(uint64_t) * // indices are 64 bits - 2; // 2 indices per chunk - const auto expected_file_size = - shard_width * shard_height * shard_planes * chunk_size + - index_size; - - const fs::path data_root = - base_dir / "data/root" / std::to_string(config.level_of_detail); - CHECK(fs::is_directory(data_root)); - for (auto z = 0; z < shards_in_z; ++z) { - const auto z_dir = data_root / ("c" + std::to_string(z)); - CHECK(fs::is_directory(z_dir)); - - for (auto y = 0; y < shards_in_y; ++y) { - const auto y_dir = z_dir / std::to_string(y); - CHECK(fs::is_directory(y_dir)); - - for (auto x = 0; x < shards_in_x; ++x) { - const auto x_file = y_dir / std::to_string(x); - CHECK(fs::is_regular_file(x_file)); - const auto file_size = fs::file_size(x_file); - CHECK(file_size == expected_file_size); - } - - CHECK(!fs::is_regular_file(y_dir / - std::to_string(shards_in_x))); - } - - CHECK(!fs::is_directory(z_dir / std::to_string(shards_in_y))); - } - - CHECK(!fs::is_directory(base_dir / - ("c" + std::to_string(shards_in_z)))); - - retval = 1; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - - // cleanup - if (fs::exists(base_dir)) { - fs::remove_all(base_dir); - } - return retval; - } - - acquire_export int unit_test__zarrv3_writer__write_ragged_internal_dim() - { - int retval = 0; - const fs::path base_dir = fs::temp_directory_path() / "acquire"; - - const unsigned int array_width = 64, array_height = 48, - array_planes = 5, array_timepoints = 10; - const unsigned int n_frames = array_planes * array_timepoints; - - const unsigned int chunk_width = 16, chunk_height = 16, - chunk_planes = 2, chunk_timepoints = 5; - - const unsigned int shard_width = 2, shard_height = 1, shard_planes = 1, - shard_timepoints = 2; - const unsigned int chunks_per_shard = - shard_width * shard_height * shard_planes * shard_timepoints; - - const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks - const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks - const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks, ragged - const unsigned int chunks_in_t = - (array_timepoints + chunk_timepoints - 1) / chunk_timepoints; - - const unsigned int shards_in_x = - (chunks_in_x + shard_width - 1) / shard_width; // 2 shards - const unsigned int shards_in_y = - (chunks_in_y + shard_height - 1) / shard_height; // 3 shards - const unsigned int shards_in_z = - (chunks_in_z + shard_planes - 1) / shard_planes; // 3 shards - const unsigned int shards_in_t = - (chunks_in_t + shard_timepoints - 1) / shard_timepoints; // 1 shard - - const ImageShape shape - { - .dims = { - .width = array_width, - .height = array_height, - }, - .strides = { - .width = 1, - .height = array_width, - .planes = array_width * array_height, - }, - .type = SampleType_f32, - }; - const unsigned int nbytes_px = bytes_of_type(shape.type); - - try { - auto thread_pool = std::make_shared( - std::thread::hardware_concurrency(), - [](const std::string& err) { LOGE("Error: %s", err.c_str()); }); - - std::vector dims; - dims.emplace_back( - "x", DimensionType_Space, array_width, chunk_width, shard_width); - dims.emplace_back("y", - DimensionType_Space, - array_height, - chunk_height, - shard_height); - dims.emplace_back("z", - DimensionType_Space, - array_planes, - chunk_planes, - shard_planes); - dims.emplace_back("t", - DimensionType_Time, - array_timepoints, - chunk_timepoints, - shard_timepoints); - - zarr::ArrayWriterConfig config = { - .image_shape = shape, - .dimensions = dims, - .level_of_detail = 5, - .dataset_root = base_dir.string(), - .compression_params = std::nullopt, - }; - - zarr::ZarrV3ArrayWriter writer( - config, thread_pool, std::shared_ptr()); - - const size_t frame_size = array_width * array_height * nbytes_px; - std::vector data(frame_size, 0); - - for (auto i = 0; i < n_frames; ++i) { - CHECK(writer.write(data.data(), frame_size) == frame_size); - } - writer.finalize(); - - const auto chunk_size = chunk_width * chunk_height * chunk_planes * - chunk_timepoints * nbytes_px; - const auto index_size = chunks_per_shard * - sizeof(uint64_t) * // indices are 64 bits - 2; // 2 indices per chunk - const auto expected_file_size = shard_width * shard_height * - shard_planes * shard_timepoints * - chunk_size + - index_size; - - const fs::path data_root = - base_dir / "data/root" / std::to_string(config.level_of_detail); - CHECK(fs::is_directory(data_root)); - for (auto t = 0; t < shards_in_t; ++t) { - const auto t_dir = data_root / ("c" + std::to_string(t)); - CHECK(fs::is_directory(t_dir)); - - for (auto z = 0; z < shards_in_z; ++z) { - const auto z_dir = t_dir / std::to_string(z); - CHECK(fs::is_directory(z_dir)); - - for (auto y = 0; y < shards_in_y; ++y) { - const auto y_dir = z_dir / std::to_string(y); - CHECK(fs::is_directory(y_dir)); - - for (auto x = 0; x < shards_in_x; ++x) { - const auto x_file = y_dir / std::to_string(x); - CHECK(fs::is_regular_file(x_file)); - const auto file_size = fs::file_size(x_file); - CHECK(file_size == expected_file_size); - } - - CHECK(!fs::is_regular_file( - y_dir / std::to_string(shards_in_x))); - } - - CHECK( - !fs::is_directory(z_dir / std::to_string(shards_in_y))); - } - - CHECK(!fs::is_directory(t_dir / std::to_string(shards_in_z))); - } - - CHECK(!fs::is_directory(base_dir / - ("c" + std::to_string(shards_in_t)))); - - retval = 1; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - - // cleanup - if (fs::exists(base_dir)) { - fs::remove_all(base_dir); - } - return retval; - } -} -#endif diff --git a/src/driver/writers/zarrv3.array.writer.hh b/src/driver/writers/zarrv3.array.writer.hh deleted file mode 100644 index ecb81d32..00000000 --- a/src/driver/writers/zarrv3.array.writer.hh +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -#include "array.writer.hh" - -#include "platform.h" -#include "device/props/components.h" - -#include -#include -#include -#include -#include -#include - -namespace fs = std::filesystem; - -namespace acquire::sink::zarr { -struct ZarrV3ArrayWriter final : public ArrayWriter -{ - public: - ZarrV3ArrayWriter() = delete; - ZarrV3ArrayWriter( - const ArrayWriterConfig& array_spec, - std::shared_ptr thread_pool, - std::shared_ptr connection_pool); - - ~ZarrV3ArrayWriter() override = default; - - private: - std::vector shard_file_offsets_; - std::vector> shard_tables_; - - bool flush_impl_() override; - bool write_array_metadata_() override; - bool should_rollover_() const override; -}; -} // namespace acquire::sink::zarr diff --git a/src/driver/zarr.cpp b/src/driver/zarr.cpp deleted file mode 100644 index be85d613..00000000 --- a/src/driver/zarr.cpp +++ /dev/null @@ -1,937 +0,0 @@ -#include "zarr.hh" - -#include "writers/zarrv2.array.writer.hh" -#include "nlohmann/json.hpp" - -#include // std::ignore - -namespace zarr = acquire::sink::zarr; -namespace common = zarr::common; -using json = nlohmann::json; - -namespace { -/// \brief Get the filename from a StorageProperties as fs::path. -/// \param props StorageProperties for the Zarr Storage device. -/// \return fs::path representation of the Zarr data directory. -fs::path -as_path(const StorageProperties& props) -{ - if (!props.uri.str) { - return {}; - } - - std::string uri{ props.uri.str, props.uri.nbytes - 1 }; - - if (uri.find("file://") == std::string::npos) { - return uri; - } - - return uri.substr(7); // strlen("file://") == 7 -} - -/// \brief Check that the JSON string is valid. (Valid can mean empty.) -/// \param str Putative JSON metadata string. -/// \param nbytes Size of the JSON metadata char array -void -validate_json(const char* str, size_t nbytes) -{ - // Empty strings are valid (no metadata is fine). - if (nbytes <= 1 || nullptr == str) { - return; - } - - std::ignore = json::parse(str, - str + nbytes, - nullptr, // callback - true, // allow exceptions - true // ignore comments - ); -} - -/// \brief Check that the StorageProperties are valid. -/// \details Assumes either an empty or valid JSON metadata string and a -/// filename string that points to a writable directory. \param props Storage -/// properties for Zarr. \throw std::runtime_error if the parent of the Zarr -/// data directory is not an existing directory. -void -validate_props(const StorageProperties* props) -{ - EXPECT(props->uri.str, "URI string is NULL."); - EXPECT(props->uri.nbytes, "URI string is zero size."); - - // check that JSON is correct (throw std::exception if not) - validate_json(props->external_metadata_json.str, - props->external_metadata_json.nbytes); - - std::string uri{ props->uri.str, props->uri.nbytes - 1 }; - - if (common::is_web_uri(uri)) { - std::vector tokens = common::split_uri(uri); - CHECK(tokens.size() > 2); // http://endpoint/bucket - } else { - const fs::path path = as_path(*props); - fs::path parent_path = path.parent_path(); - if (parent_path.empty()) - parent_path = "."; - - EXPECT(fs::is_directory(parent_path), - "Expected \"%s\" to be a directory.", - parent_path.string().c_str()); - - // check directory is writable - const auto perms = fs::status(fs::path(parent_path)).permissions(); - - EXPECT((perms & (fs::perms::owner_write | fs::perms::group_write | - fs::perms::others_write)) != fs::perms::none, - "Expected \"%s\" to have write permissions.", - parent_path.c_str()); - } -} - -void -validate_dimension(const zarr::Dimension& dim, bool is_append) -{ - if (is_append) { - EXPECT(dim.array_size_px == 0, - "Append dimension array size must be 0."); - } else { - EXPECT(dim.array_size_px > 0, "Dimension array size must be positive."); - } - - EXPECT(dim.chunk_size_px > 0, "Dimension chunk size must be positive."); -} - -[[nodiscard]] bool -is_multiscale_supported(const std::vector& dims) -{ - // 0. Must have at least 3 dimensions. - if (dims.size() < 3) { - return false; - } - - // 1. The first two dimensions must be space dimensions. - if (dims.at(0).kind != DimensionType_Space || - dims.at(1).kind != DimensionType_Space) { - return false; - } - - // 2. Interior dimensions must have size 1 - for (auto i = 2; i < dims.size() - 1; ++i) { - if (dims.at(i).array_size_px != 1) { - return false; - } - } - - return true; -} - -template -VideoFrame* -scale_image(const uint8_t* const data, - size_t bytes_of_data, - const struct ImageShape& shape) -{ - CHECK(data); - CHECK(bytes_of_data); - - const int downscale = 2; - constexpr size_t bytes_of_type = sizeof(T); - const auto factor = 0.25f; - - const auto width = shape.dims.width; - const auto w_pad = width + (width % downscale); - - const auto height = shape.dims.height; - const auto h_pad = height + (height % downscale); - - const auto size_of_image = - static_cast(w_pad * h_pad * factor * bytes_of_type); - - const size_t bytes_of_frame = - common::align_up(sizeof(VideoFrame) + size_of_image, 8); - - auto* dst = (VideoFrame*)malloc(bytes_of_frame); - CHECK(dst); - dst->bytes_of_frame = bytes_of_frame; - - { - dst->shape = shape; - dst->shape.dims = { - .width = w_pad / downscale, - .height = h_pad / downscale, - }; - dst->shape.strides = { - .height = dst->shape.dims.width, - .planes = dst->shape.dims.width * dst->shape.dims.height, - }; - - CHECK(bytes_of_image(&dst->shape) == size_of_image); - } - - const auto* src_img = (T*)data; - auto* dst_img = (T*)dst->data; - memset(dst_img, 0, size_of_image); - - size_t dst_idx = 0; - for (auto row = 0; row < height; row += downscale) { - const bool pad_height = (row == height - 1 && height != h_pad); - - for (auto col = 0; col < width; col += downscale) { - const bool pad_width = (col == width - 1 && width != w_pad); - - size_t idx = row * width + col; - dst_img[dst_idx++] = - (T)(factor * - ((float)src_img[idx] + - (float)src_img[idx + (1 - (int)pad_width)] + - (float)src_img[idx + width * (1 - (int)pad_height)] + - (float)src_img[idx + width * (1 - (int)pad_height) + - (1 - (int)pad_width)])); - } - } - - return dst; -} - -/// @brief Average both `dst` and `src` into `dst`. -template -void -average_two_frames(VideoFrame* dst, const VideoFrame* src) -{ - CHECK(dst); - CHECK(src); - CHECK(dst->bytes_of_frame == src->bytes_of_frame); - - const auto nbytes_image = bytes_of_image(&dst->shape); - const auto num_pixels = nbytes_image / sizeof(T); - for (auto i = 0; i < num_pixels; ++i) { - dst->data[i] = (T)(0.5f * ((float)dst->data[i] + (float)src->data[i])); - } -} - -DeviceState -zarr_set(Storage* self_, const StorageProperties* props) noexcept -{ - try { - CHECK(self_); - auto* self = (zarr::Zarr*)self_; - self->set(props); - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - return DeviceState_AwaitingConfiguration; - } catch (...) { - LOGE("Exception: (unknown)"); - return DeviceState_AwaitingConfiguration; - } - - return DeviceState_Armed; -} - -void -zarr_get(const Storage* self_, StorageProperties* props) noexcept -{ - try { - CHECK(self_); - auto* self = (zarr::Zarr*)self_; - self->get(props); - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } -} - -void -zarr_get_meta(const Storage* self_, StoragePropertyMetadata* meta) noexcept -{ - try { - CHECK(self_); - auto* self = (zarr::Zarr*)self_; - self->get_meta(meta); - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } -} - -DeviceState -zarr_start(Storage* self_) noexcept -{ - DeviceState state{ DeviceState_AwaitingConfiguration }; - try { - CHECK(self_); - auto* self = (zarr::Zarr*)self_; - self->start(); - state = self->state; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - - return state; -} - -DeviceState -zarr_append(Storage* self_, const VideoFrame* frames, size_t* nbytes) noexcept -{ - DeviceState state{ DeviceState_AwaitingConfiguration }; - try { - CHECK(self_); - auto* self = (zarr::Zarr*)self_; - *nbytes = self->append(frames, *nbytes); - state = self->state; - } catch (const std::exception& exc) { - *nbytes = 0; - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - *nbytes = 0; - LOGE("Exception: (unknown)"); - } - - return state; -} - -DeviceState -zarr_stop(Storage* self_) noexcept -{ - DeviceState state{ DeviceState_AwaitingConfiguration }; - - try { - CHECK(self_); - auto* self = (zarr::Zarr*)self_; - CHECK(self->stop()); // state is set to DeviceState_Armed here - state = self->state; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - - return state; -} - -void -zarr_destroy(Storage* self_) noexcept -{ - try { - CHECK(self_); - auto* self = (zarr::Zarr*)self_; - if (self_->stop) - self_->stop(self_); - - delete self; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } -} - -void -zarr_reserve_image_shape(Storage* self_, const ImageShape* shape) noexcept -{ - try { - CHECK(self_); - auto* self = (zarr::Zarr*)self_; - self->reserve_image_shape(shape); - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } -} -} // end ::{anonymous} namespace - -void -zarr::Zarr::set(const StorageProperties* props) -{ - EXPECT(state != DeviceState_Running, - "Cannot set properties while running."); - CHECK(props); - - // checks the directory exists and is writable - validate_props(props); - - std::string uri(props->uri.str, props->uri.nbytes - 1); - - if (common::is_web_uri(uri)) { - dataset_root_ = uri; - } else { - dataset_root_ = as_path(*props).string(); - } - - if (props->access_key_id.str) { - s3_access_key_id_ = std::string(props->access_key_id.str, - props->access_key_id.nbytes - 1); - } - - if (props->secret_access_key.str) { - s3_secret_access_key_ = std::string( - props->secret_access_key.str, props->secret_access_key.nbytes - 1); - } - - if (props->external_metadata_json.str) { - external_metadata_json_ = - std::string(props->external_metadata_json.str, - props->external_metadata_json.nbytes - 1); - } - - pixel_scale_um_ = props->pixel_scale_um; - - set_dimensions_(props); - enable_multiscale_ = props->enable_multiscale && - is_multiscale_supported(acquisition_dimensions_); -} - -void -zarr::Zarr::get(StorageProperties* props) const -{ - CHECK(props); - storage_properties_destroy(props); - - std::string uri; - if (common::is_web_uri(dataset_root_)) { - uri = dataset_root_; - } else if (!dataset_root_.empty()) { - fs::path dataset_root_abs = fs::absolute(dataset_root_); - uri = "file://" + dataset_root_abs.string(); - } - - const size_t bytes_of_filename = uri.empty() ? 0 : uri.size() + 1; - - const char* metadata = external_metadata_json_.empty() - ? nullptr - : external_metadata_json_.c_str(); - const size_t bytes_of_metadata = - metadata ? external_metadata_json_.size() + 1 : 0; - - CHECK(storage_properties_init(props, - 0, - uri.c_str(), - bytes_of_filename, - metadata, - bytes_of_metadata, - pixel_scale_um_, - acquisition_dimensions_.size())); - - // set access key and secret - { - const char* access_key_id = - s3_access_key_id_.has_value() ? s3_access_key_id_->c_str() : nullptr; - const size_t bytes_of_access_key_id = - access_key_id ? s3_access_key_id_->size() + 1 : 0; - - const char* secret_access_key = s3_secret_access_key_.has_value() - ? s3_secret_access_key_->c_str() - : nullptr; - const size_t bytes_of_secret_access_key = - secret_access_key ? s3_secret_access_key_->size() + 1 : 0; - - if (access_key_id && secret_access_key) { - CHECK(storage_properties_set_access_key_and_secret( - props, - access_key_id, - bytes_of_access_key_id, - secret_access_key, - bytes_of_secret_access_key)); - } - } - - for (auto i = 0; i < acquisition_dimensions_.size(); ++i) { - const auto dim = acquisition_dimensions_.at(i); - CHECK(storage_properties_set_dimension(props, - i, - dim.name.c_str(), - dim.name.length() + 1, - dim.kind, - dim.array_size_px, - dim.chunk_size_px, - dim.shard_size_chunks)); - } - - storage_properties_set_enable_multiscale(props, - (uint8_t)enable_multiscale_); -} - -void -zarr::Zarr::get_meta(StoragePropertyMetadata* meta) const -{ - CHECK(meta); - memset(meta, 0, sizeof(*meta)); - - meta->chunking_is_supported = 1; - meta->multiscale_is_supported = 1; - meta->s3_is_supported = 1; -} - -void -zarr::Zarr::start() -{ - error_ = true; - - thread_pool_ = std::make_shared( - std::thread::hardware_concurrency(), - [this](const std::string& err) { this->set_error(err); }); - - if (common::is_web_uri(dataset_root_)) { - std::vector tokens = common::split_uri(dataset_root_); - CHECK(tokens.size() > 1); - const std::string endpoint = tokens[0] + "//" + tokens[1]; - connection_pool_ = std::make_shared( - 8, endpoint, *s3_access_key_id_, *s3_secret_access_key_); - } else { - // remove the folder if it exists - if (fs::exists(dataset_root_)) { - std::error_code ec; - EXPECT(fs::remove_all(dataset_root_, ec), - R"(Failed to remove folder for "%s": %s)", - dataset_root_.c_str(), - ec.message().c_str()); - } - - // create the dataset folder - fs::create_directories(dataset_root_); - } - - allocate_writers_(); - - make_metadata_sinks_(); - write_fixed_metadata_(); - - state = DeviceState_Running; - error_ = false; -} - -int -zarr::Zarr::stop() noexcept -{ - int is_ok = 1; - - if (DeviceState_Running == state) { - state = DeviceState_Armed; - is_ok = 0; - - try { - // must precede close of chunk file - write_group_metadata_(); - metadata_sinks_.clear(); - - for (auto& writer : writers_) { - writer->finalize(); - } - - // call await_stop() before destroying to give jobs a chance to - // finish - thread_pool_->await_stop(); - thread_pool_ = nullptr; - - connection_pool_ = nullptr; - - // don't clear before all working threads have shut down - writers_.clear(); - - // should be empty, but just in case - for (auto& [_, frame] : scaled_frames_) { - if (frame && *frame) { - free(*frame); - } - } - scaled_frames_.clear(); - - error_ = false; - error_msg_.clear(); - - is_ok = 1; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - } - - return is_ok; -} - -size_t -zarr::Zarr::append(const VideoFrame* frames, size_t nbytes) -{ - CHECK(DeviceState_Running == state); - EXPECT(!error_, "%s", error_msg_.c_str()); - - if (0 == nbytes) { - return nbytes; - } - - const VideoFrame* cur = nullptr; - const auto* end = (const VideoFrame*)((uint8_t*)frames + nbytes); - auto next = [&]() -> const VideoFrame* { - const uint8_t* p = ((const uint8_t*)cur) + cur->bytes_of_frame; - return (const VideoFrame*)p; - }; - - for (cur = frames; cur < end; cur = next()) { - const size_t bytes_of_frame = bytes_of_image(&cur->shape); - const size_t bytes_written = append_frame( - const_cast(cur->data), bytes_of_frame, cur->shape); - EXPECT(bytes_written == bytes_of_frame, - "Expected to write %zu bytes, but wrote %zu.", - bytes_of_frame, - bytes_written); - } - - return nbytes; -} - -size_t -zarr::Zarr::append_frame(const uint8_t* const data, - size_t bytes_of_data, - const ImageShape& shape) -{ - CHECK(DeviceState_Running == state); - EXPECT(!error_, "%s", error_msg_.c_str()); - - if (!data || !bytes_of_data) { - return 0; - } - - const size_t bytes_written = writers_.at(0)->write(data, bytes_of_data); - if (bytes_written != bytes_of_data) { - set_error("Failed to write frame."); - return bytes_written; - } - - // multiscale - if (writers_.size() > 1) { - write_multiscale_frames_(data, bytes_written, shape); - } - - return bytes_written; -} - -void -zarr::Zarr::reserve_image_shape(const ImageShape* shape) -{ - EXPECT(state != DeviceState_Running, - "Cannot reserve image shape while running."); - - // `shape` should be verified nonnull in storage_reserve_image_shape, but - // let's check anyway - CHECK(shape); - - // image shape should be compatible with first two acquisition dimensions - EXPECT(shape->dims.width == acquisition_dimensions_.at(0).array_size_px, - "Image width must match first acquisition dimension."); - EXPECT(shape->dims.height == acquisition_dimensions_.at(1).array_size_px, - "Image height must match second acquisition dimension."); - - image_shape_ = *shape; -} - -/// Zarr - -zarr::Zarr::Zarr() - : Storage { - .state = DeviceState_AwaitingConfiguration, - .set = ::zarr_set, - .get = ::zarr_get, - .get_meta = ::zarr_get_meta, - .start = ::zarr_start, - .append = ::zarr_append, - .stop = ::zarr_stop, - .destroy = ::zarr_destroy, - .reserve_image_shape = ::zarr_reserve_image_shape, - } - , thread_pool_{ nullptr } - , pixel_scale_um_{ 1, 1 } - , enable_multiscale_{ false } - , error_{ false } -{ -} - -zarr::Zarr::Zarr(BloscCompressionParams&& compression_params) - : Zarr() -{ - blosc_compression_params_ = std::move(compression_params); -} - -void -zarr::Zarr::set_dimensions_(const StorageProperties* props) -{ - const auto dimension_count = props->acquisition_dimensions.size; - EXPECT(dimension_count > 2, "Expected at least 3 dimensions."); - - acquisition_dimensions_.clear(); - - for (auto i = 0; i < dimension_count; ++i) { - CHECK(props->acquisition_dimensions.data[i].name.str); - Dimension dim(props->acquisition_dimensions.data[i]); - validate_dimension(dim, i == dimension_count - 1); - - acquisition_dimensions_.push_back(dim); - } -} - -void -zarr::Zarr::set_error(const std::string& msg) noexcept -{ - std::scoped_lock lock(mutex_); - - // don't overwrite the first error - if (!error_) { - error_ = true; - error_msg_ = msg; - } -} - -void -zarr::Zarr::write_fixed_metadata_() const -{ - write_base_metadata_(); - write_external_metadata_(); -} - -json -zarr::Zarr::make_multiscale_metadata_() const -{ - json multiscales = json::array({ json::object() }); - // write multiscale metadata - multiscales[0]["version"] = "0.4"; - - auto& axes = multiscales[0]["axes"]; - for (auto dim = acquisition_dimensions_.rbegin(); - dim != acquisition_dimensions_.rend(); - ++dim) { - std::string type; - switch (dim->kind) { - case DimensionType_Space: - type = "space"; - break; - case DimensionType_Channel: - type = "channel"; - break; - case DimensionType_Time: - type = "time"; - break; - case DimensionType_Other: - type = "other"; - break; - default: - throw std::runtime_error("Unknown dimension type"); - } - - if (dim < acquisition_dimensions_.rend() - 2) { - axes.push_back({ { "name", dim->name }, { "type", type } }); - } else { - axes.push_back({ { "name", dim->name }, - { "type", type }, - { "unit", "micrometer" } }); - } - } - - // spatial multiscale metadata - if (writers_.empty()) { - std::vector scales; - for (auto i = 0; i < acquisition_dimensions_.size() - 2; ++i) { - scales.push_back(1.); - } - scales.push_back(pixel_scale_um_.y); - scales.push_back(pixel_scale_um_.x); - - multiscales[0]["datasets"] = { - { - { "path", "0" }, - { "coordinateTransformations", - { - { - { "type", "scale" }, - { "scale", scales }, - }, - } - }, - }, - }; - } else { - for (auto i = 0; i < writers_.size(); ++i) { - std::vector scales; - scales.push_back(std::pow(2, i)); // append - for (auto k = 0; k < acquisition_dimensions_.size() - 3; ++k) { - scales.push_back(1.); - } - scales.push_back(std::pow(2, i) * pixel_scale_um_.y); // y - scales.push_back(std::pow(2, i) * pixel_scale_um_.x); // x - - multiscales[0]["datasets"].push_back({ - { "path", std::to_string(i) }, - { "coordinateTransformations", - { - { - { "type", "scale" }, - { "scale", scales }, - }, - } - }, - }); - } - - // downsampling metadata - multiscales[0]["type"] = "local_mean"; - multiscales[0]["metadata"] = { - { "description", - "The fields in the metadata describe how to reproduce this " - "multiscaling in scikit-image. The method and its parameters are " - "given here." }, - { "method", "skimage.transform.downscale_local_mean" }, - { "version", "0.21.0" }, - { "args", "[2]" }, - { "kwargs", { "cval", 0 } }, - }; - } - - return multiscales; -} - -void -zarr::Zarr::write_multiscale_frames_(const uint8_t* const data_, - size_t bytes_of_data, - const ImageShape& shape_) -{ - auto* data = const_cast(data_); - ImageShape shape = shape_; - struct VideoFrame* dst; - - std::function scale; - std::function average2; - switch (shape.type) { - case SampleType_u10: - case SampleType_u12: - case SampleType_u14: - case SampleType_u16: - scale = ::scale_image; - average2 = ::average_two_frames; - break; - case SampleType_i8: - scale = ::scale_image; - average2 = ::average_two_frames; - break; - case SampleType_i16: - scale = ::scale_image; - average2 = ::average_two_frames; - break; - case SampleType_f32: - scale = ::scale_image; - average2 = ::average_two_frames; - break; - case SampleType_u8: - scale = ::scale_image; - average2 = ::average_two_frames; - break; - default: - char err_msg[64]; - snprintf(err_msg, - sizeof(err_msg), - "Unsupported pixel type: %s", - common::sample_type_to_string(shape.type)); - throw std::runtime_error(err_msg); - } - - for (auto i = 1; i < writers_.size(); ++i) { - dst = scale(data, bytes_of_data, shape); - if (scaled_frames_.at(i).has_value()) { - // average - average2(dst, scaled_frames_.at(i).value()); - - // write the downsampled frame - const size_t bytes_of_frame = bytes_of_image(&dst->shape); - CHECK(writers_.at(i)->write(dst->data, bytes_of_frame)); - - // clean up this level of detail - free(scaled_frames_.at(i).value()); - scaled_frames_.at(i).reset(); - - // setup for next iteration - if (i + 1 < writers_.size()) { - data = dst->data; - shape = dst->shape; - bytes_of_data = bytes_of_image(&shape); - } else { - // no longer needed - free(dst); - } - } else { - scaled_frames_.at(i) = dst; - break; - } - } -} - -#ifndef NO_UNIT_TESTS -#ifdef _WIN32 -#define acquire_export __declspec(dllexport) -#else -#define acquire_export -#endif - -///< Test that a single frame with 1 plane is padded and averaged correctly. -template -void -test_average_frame_inner(const SampleType& stype) -{ - const size_t bytes_of_frame = - common::align_up(sizeof(VideoFrame) + 9 * sizeof(T), 8); - auto* src = (VideoFrame*)malloc(bytes_of_frame); - CHECK(src); - - src->bytes_of_frame = bytes_of_frame; - src->shape = { - .dims = { - .channels = 1, - .width = 3, - .height = 3, - .planes = 1, - }, - .strides = { - .channels = 1, - .width = 1, - .height = 3, - .planes = 9 - }, - .type = stype - }; - - for (auto i = 0; i < 9; ++i) { - ((T*)src->data)[i] = (T)(i + 1); - } - - auto dst = - scale_image(src->data, bytes_of_image(&src->shape), src->shape); - CHECK(((T*)dst->data)[0] == (T)3); - CHECK(((T*)dst->data)[1] == (T)4.5); - CHECK(((T*)dst->data)[2] == (T)7.5); - CHECK(((T*)dst->data)[3] == (T)9); - - free(src); - free(dst); -} - -extern "C" acquire_export int -unit_test__average_frame() -{ - try { - test_average_frame_inner(SampleType_u8); - test_average_frame_inner(SampleType_i8); - test_average_frame_inner(SampleType_u16); - test_average_frame_inner(SampleType_i16); - test_average_frame_inner(SampleType_f32); - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - return 0; - } catch (...) { - LOGE("Exception: (unknown)"); - return 0; - } - - return 1; -} -#endif diff --git a/src/driver/zarr.hh b/src/driver/zarr.hh deleted file mode 100644 index bba1c392..00000000 --- a/src/driver/zarr.hh +++ /dev/null @@ -1,104 +0,0 @@ -#ifndef H_ACQUIRE_STORAGE_ZARR_V0 -#define H_ACQUIRE_STORAGE_ZARR_V0 - -#include "device/kit/storage.h" - -#include "common/utilities.hh" -#include "common/thread.pool.hh" -#include "common/s3.connection.hh" -#include "writers/array.writer.hh" -#include "writers/blosc.compressor.hh" - -#include -#include -#include -#include -#include // std::pair -#include - -#include - -namespace fs = std::filesystem; -using json = nlohmann::json; - -namespace acquire::sink::zarr { -struct Zarr : public Storage -{ - public: - Zarr(); - explicit Zarr(BloscCompressionParams&& compression_params); - virtual ~Zarr() noexcept = default; - - /// Storage interface - void set(const StorageProperties* props); - void get(StorageProperties* props) const; - virtual void get_meta(StoragePropertyMetadata* meta) const; - void start(); - int stop() noexcept; - size_t append(const VideoFrame* frames, size_t nbytes); - size_t append_frame(const uint8_t* data, - size_t bytes_of_data, - const ImageShape& shape); - void reserve_image_shape(const ImageShape* shape); - - /// Error state - void set_error(const std::string& msg) noexcept; - - protected: - /// static - set on construction - std::optional blosc_compression_params_; - - /// changes on set - std::string dataset_root_; - std::optional s3_access_key_id_; - std::optional s3_secret_access_key_; - std::string external_metadata_json_; - PixelScale pixel_scale_um_; - bool enable_multiscale_; - - /// changes on reserve_image_shape - struct ImageShape image_shape_; - std::vector acquisition_dimensions_; - std::vector> writers_; - - /// changes on append - // scaled frames, keyed by level-of-detail - std::unordered_map> scaled_frames_; - - /// changes on start - std::shared_ptr thread_pool_; - std::shared_ptr connection_pool_; - std::unordered_map> metadata_sinks_; - - /// Multithreading - mutable std::mutex mutex_; // for error_ / error_msg_ - - /// Error state - bool error_; - std::string error_msg_; - - /// Setup - void set_dimensions_(const StorageProperties* props); - virtual void allocate_writers_() = 0; - - /// Metadata - virtual void make_metadata_sinks_() = 0; - - // fixed metadata - void write_fixed_metadata_() const; - virtual void write_base_metadata_() const = 0; - virtual void write_external_metadata_() const = 0; - - // mutable metadata, changes on flush - virtual void write_group_metadata_() const = 0; - - /// Multiscale - json make_multiscale_metadata_() const; - void write_multiscale_frames_(const uint8_t* data_, - size_t bytes_of_data, - const ImageShape& shape_); -}; - -} // namespace acquire::sink::zarr - -#endif // H_ACQUIRE_STORAGE_ZARR_V0 diff --git a/src/driver/zarr.storage.cpp b/src/driver/zarr.storage.cpp new file mode 100644 index 00000000..5240ea82 --- /dev/null +++ b/src/driver/zarr.storage.cpp @@ -0,0 +1,931 @@ +#include "zarr.storage.hh" +#include "logger.h" + +#include + +#include +#include + +#define LOG(...) aq_logger(0, __FILE__, __LINE__, __FUNCTION__, __VA_ARGS__) +#define LOGE(...) aq_logger(1, __FILE__, __LINE__, __FUNCTION__, __VA_ARGS__) +#define EXPECT(e, ...) \ + do { \ + if (!(e)) { \ + LOGE(__VA_ARGS__); \ + throw std::runtime_error("Check failed: " #e); \ + } \ + } while (0) +#define CHECK(e) EXPECT(e, "Expression evaluated as false:\n\t%s", #e) + +#define ZARR_OK(e) \ + do { \ + ZarrStatusCode __err = (e); \ + EXPECT(__err == ZarrStatusCode_Success, \ + "%s", \ + Zarr_get_status_message(__err)); \ + } while (0) + +namespace sink = acquire::sink; +namespace fs = std::filesystem; + +using json = nlohmann::json; + +namespace { +/** + * @brief Align a size to a given alignment. + * @param n Size to align. + * @param align Alignment. + * @return Aligned size. + */ +size_t +align_up(size_t n, size_t align) +{ + EXPECT(align > 0, "Alignment must be greater than zero."); + return align * ((n + align - 1) / align); +} + +/** + * @brief Get the filename from a StorageProperties as fs::path. + * @param props StorageProperties for the Zarr Storage device. + * @return fs::path representation of the Zarr data directory. + */ +fs::path +as_path(const StorageProperties* props) +{ + if (!props->uri.str) { + return {}; + } + + std::string uri{ props->uri.str, props->uri.nbytes - 1 }; + + while (uri.find("file://") != std::string::npos) { + uri = uri.substr(7); // strlen("file://") == 7 + } + + return uri; +} + +/** + * @brief Check that the JSON string is valid. (Valid can mean empty.) + * @param str Putative JSON metadata string. + * @param nbytes Size of the JSON metadata char array. + */ +void +validate_json(const char* str, size_t nbytes) +{ + // Empty strings are valid (no metadata is fine). + if (nbytes <= 1 || nullptr == str) { + return; + } + + std::ignore = json::parse(str, + str + nbytes, + nullptr, // callback + true, // allow exceptions + true // ignore comments + ); +} + +/** + * @brief Check if a URI is a web URI (e.g., for S3). + * @param uri String to check. + * @return True if the URI is a web URI, false otherwise. + */ +bool +is_web_uri(std::string_view uri) +{ + return uri.starts_with("http://") || uri.starts_with("https://"); +} + +/** + * @brief Split a URI into its components. + * @param uri String to split. + * @return Vector of strings representing the components of the URI. + */ +std::vector +split_uri(const std::string& uri) +{ + const char delim = '/'; + + std::vector out; + size_t begin = 0, end = uri.find_first_of(delim); + + while (end != std::string::npos) { + std::string part = uri.substr(begin, end - begin); + if (!part.empty()) + out.push_back(part); + + begin = end + 1; + end = uri.find_first_of(delim, begin); + } + + // Add the last segment of the URI (if any) after the last '/' + std::string last_part = uri.substr(begin); + if (!last_part.empty()) { + out.push_back(last_part); + } + + return out; +} + +/// \brief Check that the StorageProperties are valid. +/// \details Assumes either an empty or valid JSON metadata string and a +/// filename string that points to a writable directory. \param props Storage +/// properties for Zarr. \throw std::runtime_error if the parent of the Zarr +/// data directory is not an existing directory. +void +validate_props(const StorageProperties* props) +{ + EXPECT(props->uri.str, "URI string is NULL."); + EXPECT(props->uri.nbytes, "URI string is zero size."); + + // check that JSON is correct (throw std::exception if not) + validate_json(props->external_metadata_json.str, + props->external_metadata_json.nbytes); + + std::string uri{ props->uri.str, props->uri.nbytes - 1 }; + + if (is_web_uri(uri)) { + std::vector tokens = split_uri(uri); + CHECK(tokens.size() > 2); // http://endpoint/bucket + } else { + const fs::path path = as_path(props); + fs::path parent_path = path.parent_path(); + if (parent_path.empty()) + parent_path = "."; + + EXPECT(fs::is_directory(parent_path), + "Expected \"%s\" to be a directory.", + parent_path.string().c_str()); + + // check directory is writable + const auto perms = fs::status(fs::path(parent_path)).permissions(); + + EXPECT((perms & (fs::perms::owner_write | fs::perms::group_write | + fs::perms::others_write)) != fs::perms::none, + "Expected \"%s\" to have write permissions.", + parent_path.c_str()); + } +} + +void +validate_dimension(const struct StorageDimension* dim, bool is_append) +{ + EXPECT(dim, "Dimension is NULL."); + + if (is_append) { + EXPECT(dim->array_size_px == 0, + "Append dimension array size must be 0."); + } else { + EXPECT(dim->array_size_px > 0, + "Dimension array size must be positive."); + } + + EXPECT(dim->chunk_size_px > 0, "Dimension chunk size must be positive."); + + EXPECT(dim->name.str, "Dimension name is NULL."); + EXPECT(dim->name.nbytes > 1, "Dimension name is empty."); +} + +[[nodiscard]] bool +is_multiscale_supported(struct StorageDimension* dims, size_t ndims) +{ + EXPECT(dims, "Dimensions are NULL."); + EXPECT(ndims > 2, "Expected at least 3 dimensions."); + + // 1. The final two dimensions must be space dimensions. + if (dims[ndims - 1].kind != DimensionType_Space || + dims[ndims - 2].kind != DimensionType_Space) { + return false; + } + + // 2. Interior dimensions must have size 1 + for (auto i = 1; i < ndims - 2; ++i) { + if (dims[i].array_size_px != 1) { + return false; + } + } + + return true; +} + +template +VideoFrame* +scale_image(const uint8_t* const data, + size_t bytes_of_data, + const struct ImageShape& shape) +{ + CHECK(data); + CHECK(bytes_of_data); + + const int downscale = 2; + constexpr size_t bytes_of_type = sizeof(T); + const auto factor = 0.25f; + + const auto width = shape.dims.width; + const auto w_pad = width + (width % downscale); + + const auto height = shape.dims.height; + const auto h_pad = height + (height % downscale); + + const auto size_of_image = + static_cast(w_pad * h_pad * factor * bytes_of_type); + + const size_t bytes_of_frame = + align_up(sizeof(VideoFrame) + size_of_image, 8); + + auto* dst = (VideoFrame*)malloc(bytes_of_frame); + CHECK(dst); + dst->bytes_of_frame = bytes_of_frame; + + { + dst->shape = shape; + dst->shape.dims = { + .width = w_pad / downscale, + .height = h_pad / downscale, + }; + dst->shape.strides = { + .height = dst->shape.dims.width, + .planes = dst->shape.dims.width * dst->shape.dims.height, + }; + + CHECK(bytes_of_image(&dst->shape) == size_of_image); + } + + const auto* src_img = (T*)data; + auto* dst_img = (T*)dst->data; + memset(dst_img, 0, size_of_image); + + size_t dst_idx = 0; + for (auto row = 0; row < height; row += downscale) { + const bool pad_height = (row == height - 1 && height != h_pad); + + for (auto col = 0; col < width; col += downscale) { + const bool pad_width = (col == width - 1 && width != w_pad); + + size_t idx = row * width + col; + dst_img[dst_idx++] = + (T)(factor * + ((float)src_img[idx] + + (float)src_img[idx + (1 - (int)pad_width)] + + (float)src_img[idx + width * (1 - (int)pad_height)] + + (float)src_img[idx + width * (1 - (int)pad_height) + + (1 - (int)pad_width)])); + } + } + + return dst; +} + +/// @brief Average both `dst` and `src` into `dst`. +template +void +average_two_frames(VideoFrame* dst, const VideoFrame* src) +{ + CHECK(dst); + CHECK(src); + CHECK(dst->bytes_of_frame == src->bytes_of_frame); + + const auto nbytes_image = bytes_of_image(&dst->shape); + const auto num_pixels = nbytes_image / sizeof(T); + for (auto i = 0; i < num_pixels; ++i) { + dst->data[i] = (T)(0.5f * ((float)dst->data[i] + (float)src->data[i])); + } +} + +DeviceState +zarr_set(Storage* self_, const StorageProperties* props) noexcept +{ + try { + CHECK(self_); + auto* self = (sink::Zarr*)self_; + self->set(props); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + return DeviceState_AwaitingConfiguration; + } catch (...) { + LOGE("Exception: (unknown)"); + return DeviceState_AwaitingConfiguration; + } + + return self_->state; +} + +void +zarr_get(const Storage* self_, StorageProperties* props) noexcept +{ + try { + CHECK(self_); + auto* self = (sink::Zarr*)self_; + self->get(props); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } +} + +void +zarr_get_meta(const Storage* self_, StoragePropertyMetadata* meta) noexcept +{ + try { + CHECK(self_); + auto* self = (sink::Zarr*)self_; + self->get_meta(meta); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } +} + +DeviceState +zarr_start(Storage* self_) noexcept +{ + DeviceState state{ DeviceState_AwaitingConfiguration }; + try { + CHECK(self_); + auto* self = (sink::Zarr*)self_; + self->start(); + state = self->state; + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } + + return state; +} + +DeviceState +zarr_append(Storage* self_, const VideoFrame* frames, size_t* nbytes) noexcept +{ + DeviceState state{ DeviceState_AwaitingConfiguration }; + try { + CHECK(self_); + auto* self = (sink::Zarr*)self_; + *nbytes = self->append(frames, *nbytes); + state = self->state; + } catch (const std::exception& exc) { + *nbytes = 0; + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + *nbytes = 0; + LOGE("Exception: (unknown)"); + } + + return state; +} + +DeviceState +zarr_stop(Storage* self_) noexcept +{ + DeviceState state{ DeviceState_AwaitingConfiguration }; + + try { + CHECK(self_); + auto* self = (sink::Zarr*)self_; + self->stop(); // state is set to DeviceState_Armed here + state = self->state; + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } + + return state; +} + +void +zarr_destroy(Storage* self_) noexcept +{ + try { + CHECK(self_); + auto* self = (sink::Zarr*)self_; + if (self_->stop) + self_->stop(self_); + + delete self; + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } +} + +void +zarr_reserve_image_shape(Storage* self_, const ImageShape* shape) noexcept +{ + try { + CHECK(self_); + auto* self = (sink::Zarr*)self_; + self->reserve_image_shape(shape); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } +} +} // end ::{anonymous} namespace + +sink::Zarr::Zarr(ZarrVersion version, + ZarrCompressionCodec compression_codec, + uint8_t compression_level, + uint8_t shuffle) + : Storage { + .state = DeviceState_AwaitingConfiguration, + .set = ::zarr_set, + .get = ::zarr_get, + .get_meta = ::zarr_get_meta, + .start = ::zarr_start, + .append = ::zarr_append, + .stop = ::zarr_stop, + .destroy = ::zarr_destroy, + .reserve_image_shape = ::zarr_reserve_image_shape, + } + , version_(version) + , store_path_() + , custom_metadata_("{}") + , dtype_(ZarrDataType_uint8) + , compression_codec_(compression_codec) + , compression_level_(compression_level) + , compression_shuffle_(shuffle) + , multiscale_(false) + , stream_(nullptr) +{ + EXPECT( + version_ < ZarrVersionCount, "Unsupported Zarr version: %d", version); + EXPECT(compression_codec_ < ZarrCompressionCodecCount, + "Unsupported compression codec: %d", + compression_codec); + EXPECT( + compression_level_ <= 9, + "Invalid compression level: %d. Compression level must be in [0, 9].", + compression_level_); + EXPECT(compression_shuffle_ <= 2, + "Invalid shuffle value: %d. Shuffle must be 0, 1, or 2.", + compression_shuffle_); +} + +sink::Zarr::~Zarr() +{ + stop(); +} + +void +sink::Zarr::set(const StorageProperties* props) +{ + EXPECT(state != DeviceState_Running, + "Cannot set properties while running."); + EXPECT(props, "StorageProperties is NULL."); + + // check that the external metadata is valid + if (props->external_metadata_json.str) { + validate_json(props->external_metadata_json.str, + props->external_metadata_json.nbytes); + + custom_metadata_ = props->external_metadata_json.str; + } + + if (custom_metadata_.empty()) { + custom_metadata_ = "{}"; + } + + EXPECT(props->uri.str, "URI string is NULL."); + EXPECT(props->uri.nbytes > 1, "URI string is empty."); + std::string uri(props->uri.str, props->uri.nbytes - 1); + + if (is_web_uri(uri)) { + EXPECT(props->access_key_id.str, "Access key ID is NULL."); + EXPECT(props->access_key_id.nbytes > 1, "Access key ID is empty."); + EXPECT(props->secret_access_key.str, "Secret access key is NULL."); + EXPECT(props->secret_access_key.nbytes > 1, + "Secret access key is empty."); + + auto components = split_uri(uri); + EXPECT(components.size() > 3, "Invalid URI: %s", uri.c_str()); + + s3_endpoint_ = components[0] + "//" + components[1]; + s3_bucket_name_ = components[2]; + s3_access_key_id_ = props->access_key_id.str; + s3_secret_access_key_ = props->secret_access_key.str; + + store_path_ = components[3]; + for (auto i = 4; i < components.size(); ++i) { + store_path_ += "/" + components[i]; + } + } else { + if (uri.find("file://") != std::string::npos) { + uri = uri.substr(7); // strlen("file://") == 7 + } + std::string store_path = uri; + + if (fs::exists(store_path)) { + std::error_code ec; + EXPECT(fs::remove_all(store_path, ec), + R"(Failed to remove folder for "%s": %s)", + store_path.c_str(), + ec.message().c_str()); + } + + fs::path parent_path = fs::path(store_path).parent_path(); + if (parent_path.empty()) + parent_path = "."; + + EXPECT(fs::is_directory(parent_path), + "Expected \"%s\" to be a directory.", + parent_path.string().c_str()); + + // check directory is writable + const auto perms = fs::status(fs::path(parent_path)).permissions(); + + EXPECT((perms & (fs::perms::owner_write | fs::perms::group_write | + fs::perms::others_write)) != fs::perms::none, + "Expected \"%s\" to have write permissions.", + parent_path.c_str()); + + store_path_ = store_path; + } + + dimension_names_.clear(); + dimensions_.clear(); + + for (auto i = 0; i < props->acquisition_dimensions.size; ++i) { + const auto* dim = props->acquisition_dimensions.data + i; + validate_dimension(dim, i == 0); + + ZarrDimensionType type; + switch (dim->kind) { + case DimensionType_Space: + type = ZarrDimensionType_Space; + break; + case DimensionType_Channel: + type = ZarrDimensionType_Channel; + break; + case DimensionType_Time: + type = ZarrDimensionType_Time; + break; + case DimensionType_Other: + type = ZarrDimensionType_Other; + break; + default: + throw std::runtime_error("Invalid dimension type: " + + std::to_string(dim->kind)); + } + + dimension_names_.emplace_back(dim->name.str); + dimensions_.emplace_back(nullptr, + type, + dim->array_size_px, + dim->chunk_size_px, + dim->shard_size_chunks); + } + + multiscale_ = props->enable_multiscale; + + state = DeviceState_Armed; +} + +void +sink::Zarr::get(StorageProperties* props) const +{ + EXPECT(props, "StorageProperties is NULL."); + + storage_properties_destroy(props); + + std::string s3_endpoint, s3_bucket; + std::string access_key_id, secret_access_key; + + size_t ndims; + + if (s3_endpoint_) { + s3_endpoint = *s3_endpoint_; + } + if (s3_bucket_name_) { + s3_bucket = *s3_bucket_name_; + } + if (s3_access_key_id_) { + access_key_id = *s3_access_key_id_; + } + if (s3_secret_access_key_) { + secret_access_key = *s3_secret_access_key_; + } + + ndims = dimension_names_.size(); + + std::string uri; + if (!s3_endpoint.empty() && !s3_bucket.empty() && !store_path_.empty()) { + uri = s3_endpoint + "/" + s3_bucket + "/" + store_path_; + } else if (!store_path_.empty()) { + uri = "file://" + fs::absolute(store_path_).string(); + } + + const size_t bytes_of_filename = uri.empty() ? 0 : uri.size() + 1; + + const char* metadata = + custom_metadata_.empty() ? nullptr : custom_metadata_.c_str(); + const size_t bytes_of_metadata = metadata ? custom_metadata_.size() + 1 : 0; + + CHECK(storage_properties_init(props, + 0, + uri.c_str(), + bytes_of_filename, + metadata, + bytes_of_metadata, + { 1, 1 }, + ndims)); + + // set access key and secret + if (!access_key_id.empty() && !secret_access_key.empty()) { + CHECK(storage_properties_set_access_key_and_secret( + props, + access_key_id.c_str(), + access_key_id.size() + 1, + secret_access_key.c_str(), + secret_access_key.size() + 1)); + } + + for (auto i = 0; i < ndims; ++i) { + ZarrDimensionType dimension_type; + size_t array_size_px, chunk_size_px, shard_size_chunks; + + auto dimension = dimensions_[i]; + + auto dim_name = dimension_names_[i]; + dimension_type = dimension.type; + array_size_px = dimension.array_size_px; + chunk_size_px = dimension.chunk_size_px; + shard_size_chunks = dimension.shard_size_chunks; + + DimensionType kind; + switch (dimension_type) { + case ZarrDimensionType_Space: + kind = DimensionType_Space; + break; + case ZarrDimensionType_Channel: + kind = DimensionType_Channel; + break; + case ZarrDimensionType_Time: + kind = DimensionType_Time; + break; + case ZarrDimensionType_Other: + kind = DimensionType_Other; + break; + default: + throw std::runtime_error("Invalid dimension type: " + + std::to_string(dimension_type)); + } + + const char* name = dim_name.empty() ? nullptr : dim_name.c_str(); + const size_t nbytes = name ? dim_name.size() + 1 : 0; + + CHECK(storage_properties_set_dimension(props, + i, + name, + nbytes, + kind, + array_size_px, + chunk_size_px, + shard_size_chunks)); + } + + CHECK(storage_properties_set_enable_multiscale(props, multiscale_)); +} + +void +sink::Zarr::get_meta(StoragePropertyMetadata* meta) const +{ + CHECK(meta); + memset(meta, 0, sizeof(*meta)); + + meta->chunking_is_supported = 1; + meta->multiscale_is_supported = 1; + meta->s3_is_supported = 1; + meta->sharding_is_supported = + static_cast(version_ == ZarrVersion_3); +} + +void +sink::Zarr::start() +{ + EXPECT(state == DeviceState_Armed, "Device is not armed."); + + if (stream_) { + ZarrStream_destroy(stream_); + stream_ = nullptr; + } + + for (auto i = 0; i < dimension_names_.size(); ++i) { + auto& dim = dimensions_[i]; + dim.name = dimension_names_[i].c_str(); + } + + ZarrStreamSettings stream_settings{ + .store_path = store_path_.c_str(), + .custom_metadata = custom_metadata_.c_str(), + .s3_settings = nullptr, + .compression_settings = nullptr, + .dimensions = dimensions_.data(), + .dimension_count = dimensions_.size(), + .multiscale = multiscale_, + .data_type = dtype_, + .version = version_, + }; + + ZarrS3Settings s3_settings; + ZarrCompressionSettings compression_settings; + + if (s3_endpoint_ && s3_bucket_name_ && s3_access_key_id_ && + s3_secret_access_key_) { + s3_settings = { .endpoint = s3_endpoint_->c_str(), + .bucket_name = s3_bucket_name_->c_str(), + .access_key_id = s3_access_key_id_->c_str(), + .secret_access_key = s3_secret_access_key_->c_str() }; + + stream_settings.s3_settings = &s3_settings; + } + + if (compression_codec_ > ZarrCompressionCodec_None) { + compression_settings = { + .compressor = ZarrCompressor_Blosc1, + .codec = compression_codec_, + .level = compression_level_, + .shuffle = compression_shuffle_, + }; + + stream_settings.compression_settings = &compression_settings; + } + + stream_ = ZarrStream_create(&stream_settings); + CHECK(stream_); + + state = DeviceState_Running; +} + +void +sink::Zarr::stop() noexcept +{ + if (DeviceState_Running == state) { + // make a copy of current settings before destroying the stream + state = DeviceState_Armed; + + ZarrStream_destroy(stream_); + stream_ = nullptr; + } +} + +size_t +sink::Zarr::append(const VideoFrame* frames, size_t nbytes) +{ + EXPECT(DeviceState_Running == state, "Device is not running."); + + if (0 == nbytes) { + return nbytes; + } + + const VideoFrame* cur = nullptr; + const auto* end = (const VideoFrame*)((uint8_t*)frames + nbytes); + auto next = [&]() -> const VideoFrame* { + const uint8_t* p = ((const uint8_t*)cur) + cur->bytes_of_frame; + return (const VideoFrame*)p; + }; + + for (cur = frames; cur < end; cur = next()) { + const size_t bytes_of_frame = bytes_of_image(&cur->shape); + size_t bytes_written; + ZARR_OK(ZarrStream_append( + stream_, cur->data, bytes_of_frame, &bytes_written)); + EXPECT(bytes_written == bytes_of_frame, + "Expected to write %zu bytes, but wrote %zu.", + bytes_of_frame, + bytes_written); + } + + return nbytes; +} + +void +sink::Zarr::reserve_image_shape(const ImageShape* shape) +{ + EXPECT(state == DeviceState_Armed, "Device is not armed."); + + // `shape` should be verified nonnull in storage_reserve_image_shape, but + // let's check anyway + CHECK(shape); + + EXPECT(dimensions_.size() > 2, "Expected at least 3 dimensions."); + + // check that the configured dimensions match the image shape + { + ZarrDimensionProperties y_dim = dimensions_[dimensions_.size() - 2]; + EXPECT(y_dim.array_size_px == shape->dims.height, + "Image height mismatch."); + + ZarrDimensionProperties x_dim = dimensions_.back(); + EXPECT(x_dim.array_size_px == shape->dims.width, + "Image width mismatch."); + } + + switch (shape->type) { + case SampleType_u8: + dtype_ = ZarrDataType_uint8; + break; + case SampleType_u10: + case SampleType_u12: + case SampleType_u14: + case SampleType_u16: + dtype_ = ZarrDataType_uint16; + break; + case SampleType_i8: + dtype_ = ZarrDataType_int8; + break; + case SampleType_i16: + dtype_ = ZarrDataType_int16; + break; + case SampleType_f32: + dtype_ = ZarrDataType_float32; + break; + default: + throw std::runtime_error("Unsupported image type: " + + std::to_string(shape->type)); + } +} + +extern "C" +{ + struct Storage* zarr_v2_init() + { + try { + return new sink::Zarr( + ZarrVersion_2, ZarrCompressionCodec_None, 0, 0); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } + return nullptr; + } + struct Storage* compressed_zarr_v2_zstd_init() + { + try { + return new sink::Zarr( + ZarrVersion_2, ZarrCompressionCodec_BloscZstd, 1, 1); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } + return nullptr; + } + + struct Storage* compressed_zarr_v2_lz4_init() + { + try { + return new sink::Zarr( + ZarrVersion_2, ZarrCompressionCodec_BloscLZ4, 1, 1); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } + return nullptr; + } + + struct Storage* zarr_v3_init() + { + try { + return new sink::Zarr( + ZarrVersion_3, ZarrCompressionCodec_None, 0, 0); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } + return nullptr; + } + struct Storage* compressed_zarr_v3_zstd_init() + { + try { + return new sink::Zarr( + ZarrVersion_3, ZarrCompressionCodec_BloscZstd, 1, 1); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } + return nullptr; + } + + struct Storage* compressed_zarr_v3_lz4_init() + { + try { + return new sink::Zarr( + ZarrVersion_3, ZarrCompressionCodec_BloscLZ4, 1, 1); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } + return nullptr; + } +} // extern "C" \ No newline at end of file diff --git a/src/driver/zarr.storage.hh b/src/driver/zarr.storage.hh new file mode 100644 index 00000000..2edf2d12 --- /dev/null +++ b/src/driver/zarr.storage.hh @@ -0,0 +1,55 @@ +#pragma once + +#include "device/kit/storage.h" + +#include "acquire.zarr.h" + +#include +#include +#include +#include + +namespace acquire::sink { +struct Zarr : public Storage +{ + public: + Zarr(ZarrVersion version, + ZarrCompressionCodec compression_codec, + uint8_t compression_level, + uint8_t shuffle); + ~Zarr(); + + /// Storage interface + void set(const StorageProperties* props); + void get(StorageProperties* props) const; + void get_meta(StoragePropertyMetadata* meta) const; + void start(); + void stop() noexcept; + size_t append(const VideoFrame* frames, size_t nbytes); + void reserve_image_shape(const ImageShape* shape); + + private: + ZarrVersion version_; + std::string store_path_; + + std::optional s3_endpoint_; + std::optional s3_bucket_name_; + std::optional s3_access_key_id_; + std::optional s3_secret_access_key_; + + std::string custom_metadata_; + + ZarrDataType dtype_; + + ZarrCompressionCodec compression_codec_; + uint8_t compression_level_; + uint8_t compression_shuffle_; + + std::vector dimension_names_; + std::vector dimensions_; + + bool multiscale_; + + ZarrStream* stream_; +}; +} // namespace acquire::sink diff --git a/src/driver/zarr.v2.cpp b/src/driver/zarr.v2.cpp deleted file mode 100644 index 6641f1ba..00000000 --- a/src/driver/zarr.v2.cpp +++ /dev/null @@ -1,152 +0,0 @@ -#include "zarr.v2.hh" -#include "writers/zarrv2.array.writer.hh" -#include "writers/sink.creator.hh" - -#include "nlohmann/json.hpp" - -#include - -namespace zarr = acquire::sink::zarr; - -namespace { -template -struct Storage* -compressed_zarr_v2_init() -{ - try { - zarr::BloscCompressionParams params( - zarr::compression_codec_as_string(), 1, 1); - return new zarr::ZarrV2(std::move(params)); - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - return nullptr; -} -} // end ::{anonymous} namespace - -/// ZarrV2 -zarr::ZarrV2::ZarrV2(BloscCompressionParams&& compression_params) - : Zarr(std::move(compression_params)) -{ -} - -void -zarr::ZarrV2::get_meta(StoragePropertyMetadata* meta) const -{ - Zarr::get_meta(meta); - meta->sharding_is_supported = 0; -} - -void -zarr::ZarrV2::allocate_writers_() -{ - writers_.clear(); - - ArrayWriterConfig config = { - .image_shape = image_shape_, - .dimensions = acquisition_dimensions_, - .level_of_detail = 0, - .dataset_root = dataset_root_, - .compression_params = blosc_compression_params_, - }; - - writers_.push_back(std::make_shared( - config, thread_pool_, connection_pool_)); - - if (enable_multiscale_) { - ArrayWriterConfig downsampled_config; - - bool do_downsample = true; - int level = 1; - while (do_downsample) { - do_downsample = downsample(config, downsampled_config); - writers_.push_back(std::make_shared( - downsampled_config, thread_pool_, connection_pool_)); - scaled_frames_.emplace(level++, std::nullopt); - - config = std::move(downsampled_config); - downsampled_config = {}; - } - } -} - -void -zarr::ZarrV2::make_metadata_sinks_() -{ - SinkCreator creator(thread_pool_, connection_pool_); - CHECK(creator.make_metadata_sinks( - ZarrVersion::V2, dataset_root_, metadata_sinks_)); -} - -void -zarr::ZarrV2::write_base_metadata_() const -{ - namespace fs = std::filesystem; - - json metadata; - metadata["multiscales"] = make_multiscale_metadata_(); - - const std::string metadata_str = metadata.dump(4); - const auto* metadata_bytes = (const uint8_t*)metadata_str.c_str(); - const std::unique_ptr& sink = metadata_sinks_.at(".zattrs"); - CHECK(sink); - CHECK(sink->write(0, metadata_bytes, metadata_str.size())); -} - -void -zarr::ZarrV2::write_external_metadata_() const -{ - namespace fs = std::filesystem; - - std::string metadata_str = external_metadata_json_.empty() - ? "{}" - : json::parse(external_metadata_json_, - nullptr, // callback - true, // allow exceptions - true // ignore comments - ) - .dump(4); - const auto* metadata_bytes = (const uint8_t*)metadata_str.c_str(); - const std::unique_ptr& sink = metadata_sinks_.at("0/.zattrs"); - CHECK(sink); - CHECK(sink->write(0, metadata_bytes, metadata_str.size())); -} - -void -zarr::ZarrV2::write_group_metadata_() const -{ - namespace fs = std::filesystem; - - const json metadata = { { "zarr_format", 2 } }; - const std::string metadata_str = metadata.dump(4); - const auto* metadata_bytes = (const uint8_t*)metadata_str.c_str(); - const std::unique_ptr& sink = metadata_sinks_.at(".zgroup"); - CHECK(sink); - CHECK(sink->write(0, metadata_bytes, metadata_str.size())); -} - -extern "C" -{ - struct Storage* zarr_v2_init() - { - try { - return new zarr::ZarrV2(); - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - return nullptr; - } - struct Storage* compressed_zarr_v2_zstd_init() - { - return compressed_zarr_v2_init(); - } - - struct Storage* compressed_zarr_v2_lz4_init() - { - return compressed_zarr_v2_init(); - } -} diff --git a/src/driver/zarr.v2.hh b/src/driver/zarr.v2.hh deleted file mode 100644 index a83bb630..00000000 --- a/src/driver/zarr.v2.hh +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef H_ACQUIRE_STORAGE_ZARR_V2_V0 -#define H_ACQUIRE_STORAGE_ZARR_V2_V0 - -#include "zarr.hh" - -namespace acquire::sink::zarr { -struct ZarrV2 final : public Zarr -{ - public: - ZarrV2() = default; - ZarrV2(BloscCompressionParams&& compression_params); - ~ZarrV2() override = default; - - /// StorageInterface - void get_meta(StoragePropertyMetadata* meta) const override; - - private: - /// Setup - void allocate_writers_() override; - - /// Metadata - void make_metadata_sinks_() override; - - // fixed metadata - void write_base_metadata_() const override; - void write_external_metadata_() const override; - - // mutable metadata, changes on flush - void write_group_metadata_() const override; -}; -} // namespace acquire::sink::zarr - -#endif // H_ACQUIRE_STORAGE_ZARR_V2_V0 diff --git a/src/driver/zarr.v3.cpp b/src/driver/zarr.v3.cpp deleted file mode 100644 index 3a35a958..00000000 --- a/src/driver/zarr.v3.cpp +++ /dev/null @@ -1,159 +0,0 @@ -#include "zarr.v3.hh" -#include "writers/zarrv3.array.writer.hh" -#include "writers/sink.creator.hh" - -#include "nlohmann/json.hpp" - -#include - -namespace zarr = acquire::sink::zarr; - -namespace { -template -struct Storage* -compressed_zarr_v3_init() -{ - try { - zarr::BloscCompressionParams params( - zarr::compression_codec_as_string(), 1, 1); - return new zarr::ZarrV3(std::move(params)); - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - return nullptr; -} -} // end ::{anonymous} namespace - -zarr::ZarrV3::ZarrV3(BloscCompressionParams&& compression_params) - : Zarr(std::move(compression_params)) -{ -} - -void -zarr::ZarrV3::allocate_writers_() -{ - writers_.clear(); - - ArrayWriterConfig config = { - .image_shape = image_shape_, - .dimensions = acquisition_dimensions_, - .level_of_detail = 0, - .dataset_root = dataset_root_, - .compression_params = blosc_compression_params_, - }; - - writers_.push_back(std::make_shared( - config, thread_pool_, connection_pool_)); - - if (enable_multiscale_) { - ArrayWriterConfig downsampled_config; - - bool do_downsample = true; - int level = 1; - while (do_downsample) { - do_downsample = downsample(config, downsampled_config); - writers_.push_back(std::make_shared( - downsampled_config, thread_pool_, connection_pool_)); - scaled_frames_.emplace(level++, std::nullopt); - - config = std::move(downsampled_config); - downsampled_config = {}; - } - } -} - -void -zarr::ZarrV3::get_meta(StoragePropertyMetadata* meta) const -{ - Zarr::get_meta(meta); - meta->sharding_is_supported = 1; -} - -void -zarr::ZarrV3::make_metadata_sinks_() -{ - SinkCreator creator(thread_pool_, connection_pool_); - CHECK(creator.make_metadata_sinks( - ZarrVersion::V3, dataset_root_, metadata_sinks_)); -} - -/// @brief Write the metadata for the dataset. -void -zarr::ZarrV3::write_base_metadata_() const -{ - namespace fs = std::filesystem; - - json metadata; - metadata["extensions"] = json::array(); - metadata["metadata_encoding"] = - "https://purl.org/zarr/spec/protocol/core/3.0"; - metadata["metadata_key_suffix"] = ".json"; - metadata["zarr_format"] = "https://purl.org/zarr/spec/protocol/core/3.0"; - - const std::string metadata_str = metadata.dump(4); - const auto* metadata_bytes = (const uint8_t*)metadata_str.c_str(); - const std::unique_ptr& sink = metadata_sinks_.at("zarr.json"); - CHECK(sink); - CHECK(sink->write(0, metadata_bytes, metadata_str.size())); -} - -/// @brief Write the external metadata. -/// @details This is a no-op for ZarrV3. Instead, external metadata is -/// stored in the group metadata. -void -zarr::ZarrV3::write_external_metadata_() const -{ - // no-op -} - -/// @brief Write the metadata for the group. -/// @details Zarr v3 stores group metadata in -/// /meta/{group_name}.group.json. We will call the group "root". -void -zarr::ZarrV3::write_group_metadata_() const -{ - namespace fs = std::filesystem; - - json metadata; - metadata["attributes"]["acquire"] = - external_metadata_json_.empty() ? "" - : json::parse(external_metadata_json_, - nullptr, // callback - true, // allow exceptions - true // ignore comments - ); - metadata["attributes"]["multiscales"] = make_multiscale_metadata_(); - - const std::string metadata_str = metadata.dump(4); - const auto* metadata_bytes = (const uint8_t*)metadata_str.c_str(); - const std::unique_ptr& sink = - metadata_sinks_.at("meta/root.group.json"); - CHECK(sink); - CHECK(sink->write(0, metadata_bytes, metadata_str.size())); -} - -extern "C" -{ - struct Storage* zarr_v3_init() - { - try { - return new zarr::ZarrV3(); - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - return nullptr; - } - struct Storage* compressed_zarr_v3_zstd_init() - { - return compressed_zarr_v3_init(); - } - - struct Storage* compressed_zarr_v3_lz4_init() - { - return compressed_zarr_v3_init(); - } -} diff --git a/src/driver/zarr.v3.hh b/src/driver/zarr.v3.hh deleted file mode 100644 index d85d6f5b..00000000 --- a/src/driver/zarr.v3.hh +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef H_ACQUIRE_STORAGE_ZARR_V3_V0 -#define H_ACQUIRE_STORAGE_ZARR_V3_V0 - -#include "zarr.hh" - -namespace acquire::sink::zarr { -struct ZarrV3 final : public Zarr -{ - public: - ZarrV3() = default; - explicit ZarrV3(BloscCompressionParams&& compression_params); - ~ZarrV3() override = default; - - /// Storage interface - void get_meta(StoragePropertyMetadata* meta) const override; - - private: - /// Setup - void allocate_writers_() override; - - /// Metadata - void make_metadata_sinks_() override; - - // fixed metadata - void write_base_metadata_() const override; - void write_external_metadata_() const override; - - // mutable metadata, changes on flush - void write_group_metadata_() const override; -}; -} // namespace acquire::sink::zarr -#endif // H_ACQUIRE_STORAGE_ZARR_V3_V0 diff --git a/src/streaming/CMakeLists.txt b/src/streaming/CMakeLists.txt index 753c9433..3ab50ace 100644 --- a/src/streaming/CMakeLists.txt +++ b/src/streaming/CMakeLists.txt @@ -51,6 +51,7 @@ target_compile_definitions(${tgt} PRIVATE set_target_properties(${tgt} PROPERTIES MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" + POSITION_INDEPENDENT_CODE ON ) install(TARGETS ${tgt} diff --git a/src/streaming/zarr.stream.hh b/src/streaming/zarr.stream.hh index d3593919..d2fd9d83 100644 --- a/src/streaming/zarr.stream.hh +++ b/src/streaming/zarr.stream.hh @@ -15,7 +15,7 @@ struct ZarrStream_s { public: - ZarrStream_s(struct ZarrStreamSettings_s* settings); + explicit ZarrStream_s(struct ZarrStreamSettings_s* settings); /** * @brief Append data to the stream. diff --git a/tests/driver/CMakeLists.txt b/tests/driver/CMakeLists.txt index b2ea8266..3fb6265d 100644 --- a/tests/driver/CMakeLists.txt +++ b/tests/driver/CMakeLists.txt @@ -12,7 +12,6 @@ set(project acquire-driver-zarr) # CMAKE_PROJECT_NAME gets overridden if this is # set(tests list-devices - unit-tests get get-meta get-set-get diff --git a/tests/driver/get-set-get.cpp b/tests/driver/get-set-get.cpp index 48ad400e..bc7a5d31 100644 --- a/tests/driver/get-set-get.cpp +++ b/tests/driver/get-set-get.cpp @@ -92,11 +92,11 @@ validate(const struct StorageProperties& props) CHECK(props.acquisition_dimensions.size == 3); CHECK(props.acquisition_dimensions.data != nullptr); - CHECK(0 == strcmp(props.acquisition_dimensions.data[0].name.str, "x")); + CHECK(0 == strcmp(props.acquisition_dimensions.data[0].name.str, "z")); CHECK(DimensionType_Space == props.acquisition_dimensions.data[0].kind); - CHECK(props.acquisition_dimensions.data[0].array_size_px == 64); - CHECK(props.acquisition_dimensions.data[0].chunk_size_px == 16); - CHECK(props.acquisition_dimensions.data[0].shard_size_chunks == 2); + CHECK(props.acquisition_dimensions.data[0].array_size_px == 0); + CHECK(props.acquisition_dimensions.data[0].chunk_size_px == 6); + CHECK(props.acquisition_dimensions.data[0].shard_size_chunks == 1); CHECK(0 == strcmp(props.acquisition_dimensions.data[1].name.str, "y")); CHECK(DimensionType_Space == props.acquisition_dimensions.data[1].kind); @@ -104,11 +104,11 @@ validate(const struct StorageProperties& props) CHECK(props.acquisition_dimensions.data[1].chunk_size_px == 16); CHECK(props.acquisition_dimensions.data[1].shard_size_chunks == 3); - CHECK(0 == strcmp(props.acquisition_dimensions.data[2].name.str, "z")); + CHECK(0 == strcmp(props.acquisition_dimensions.data[2].name.str, "x")); CHECK(DimensionType_Space == props.acquisition_dimensions.data[2].kind); - CHECK(props.acquisition_dimensions.data[2].array_size_px == 0); - CHECK(props.acquisition_dimensions.data[2].chunk_size_px == 6); - CHECK(props.acquisition_dimensions.data[2].shard_size_chunks == 1); + CHECK(props.acquisition_dimensions.data[2].array_size_px == 64); + CHECK(props.acquisition_dimensions.data[2].chunk_size_px == 16); + CHECK(props.acquisition_dimensions.data[2].shard_size_chunks == 2); CHECK(props.first_frame_id == 0); // this is ignored @@ -155,8 +155,8 @@ main() CHECK(props.uri.nbytes == 1); CHECK(props.external_metadata_json.str); - CHECK(strcmp(props.external_metadata_json.str, "") == 0); - CHECK(props.external_metadata_json.nbytes == 1); + CHECK(strcmp(props.external_metadata_json.str, "{}") == 0); + CHECK(props.external_metadata_json.nbytes == 3); CHECK(props.first_frame_id == 0); @@ -178,11 +178,11 @@ main() )); CHECK(storage_properties_set_dimension( - &props, 0, SIZED("x") + 1, DimensionType_Space, 64, 16, 2)); + &props, 0, SIZED("z") + 1, DimensionType_Space, 0, 6, 1)); CHECK(storage_properties_set_dimension( &props, 1, SIZED("y") + 1, DimensionType_Space, 48, 16, 3)); CHECK(storage_properties_set_dimension( - &props, 2, SIZED("z") + 1, DimensionType_Space, 0, 6, 1)); + &props, 2, SIZED("x") + 1, DimensionType_Space, 64, 16, 2)); props.enable_multiscale = true; diff --git a/tests/driver/get.cpp b/tests/driver/get.cpp index 8b81cba2..3a276dc2 100644 --- a/tests/driver/get.cpp +++ b/tests/driver/get.cpp @@ -97,8 +97,8 @@ main() CHECK(props.uri.nbytes == 1); CHECK(props.external_metadata_json.str); - CHECK(strcmp(props.external_metadata_json.str, "") == 0); - CHECK(props.external_metadata_json.nbytes == 1); + CHECK(strcmp(props.external_metadata_json.str, "{}") == 0); + CHECK(props.external_metadata_json.nbytes == 3); CHECK(props.first_frame_id == 0); @@ -120,11 +120,11 @@ main() )); CHECK(storage_properties_set_dimension( - &props, 0, SIZED("x") + 1, DimensionType_Space, 64, 16, 2)); + &props, 0, SIZED("z") + 1, DimensionType_Space, 0, 6, 1)); CHECK(storage_properties_set_dimension( &props, 1, SIZED("y") + 1, DimensionType_Space, 48, 16, 3)); CHECK(storage_properties_set_dimension( - &props, 2, SIZED("z") + 1, DimensionType_Space, 0, 6, 1)); + &props, 2, SIZED("x") + 1, DimensionType_Space, 64, 16, 2)); props.enable_multiscale = true; @@ -141,13 +141,13 @@ main() CHECK(props.acquisition_dimensions.data != nullptr); CHECK(0 == strcmp(props.acquisition_dimensions.data[0].name.str, - "x")); + "z")); CHECK(DimensionType_Space == props.acquisition_dimensions.data[0].kind); - CHECK(props.acquisition_dimensions.data[0].array_size_px == 64); - CHECK(props.acquisition_dimensions.data[0].chunk_size_px == 16); + CHECK(props.acquisition_dimensions.data[0].array_size_px == 0); + CHECK(props.acquisition_dimensions.data[0].chunk_size_px == 6); CHECK(props.acquisition_dimensions.data[0].shard_size_chunks == - 2); + 1); CHECK(0 == strcmp(props.acquisition_dimensions.data[1].name.str, "y")); @@ -159,13 +159,13 @@ main() 3); CHECK(0 == strcmp(props.acquisition_dimensions.data[2].name.str, - "z")); + "x")); CHECK(DimensionType_Space == props.acquisition_dimensions.data[2].kind); - CHECK(props.acquisition_dimensions.data[2].array_size_px == 0); - CHECK(props.acquisition_dimensions.data[2].chunk_size_px == 6); + CHECK(props.acquisition_dimensions.data[2].array_size_px == 64); + CHECK(props.acquisition_dimensions.data[2].chunk_size_px == 16); CHECK(props.acquisition_dimensions.data[2].shard_size_chunks == - 1); + 2); CHECK(props.first_frame_id == 0); // this is ignored diff --git a/tests/driver/metadata-dimension-sizes.cpp b/tests/driver/metadata-dimension-sizes.cpp index 014f32bc..234963a5 100644 --- a/tests/driver/metadata-dimension-sizes.cpp +++ b/tests/driver/metadata-dimension-sizes.cpp @@ -140,17 +140,17 @@ configure_storage(AcquireRuntime* runtime, CHECK(storage_properties_set_dimension(&props.settings, 0, - SIZED("x") + 1, - DimensionType_Space, - array_width, - chunk_width, + SIZED("t") + 1, + DimensionType_Time, + 0, + chunk_timepoints, 1)); CHECK(storage_properties_set_dimension(&props.settings, 1, - SIZED("y") + 1, - DimensionType_Space, - array_height, - chunk_height, + SIZED("c") + 1, + DimensionType_Channel, + array_channels, + chunk_channels, 1)); CHECK(storage_properties_set_dimension(&props.settings, 2, @@ -161,17 +161,17 @@ configure_storage(AcquireRuntime* runtime, 1)); CHECK(storage_properties_set_dimension(&props.settings, 3, - SIZED("c") + 1, - DimensionType_Channel, - array_channels, - chunk_channels, + SIZED("y") + 1, + DimensionType_Space, + array_height, + chunk_height, 1)); CHECK(storage_properties_set_dimension(&props.settings, 4, - SIZED("t") + 1, - DimensionType_Time, - 0, - chunk_timepoints, + SIZED("x") + 1, + DimensionType_Space, + array_width, + chunk_width, 1)); } diff --git a/tests/driver/multiscales-metadata.cpp b/tests/driver/multiscales-metadata.cpp index 44c17b4a..8a63d78a 100644 --- a/tests/driver/multiscales-metadata.cpp +++ b/tests/driver/multiscales-metadata.cpp @@ -88,8 +88,6 @@ setup(AcquireRuntime* runtime, const char* filename) AcquireProperties props = {}; OK(acquire_get_configuration(runtime, &props)); - struct PixelScale sample_spacing_um = { .x = 1.0f, .y = 2.0f }; - DEVOK(device_manager_select(dm, DeviceKind_Camera, SIZED("simulated.*empty.*"), @@ -105,15 +103,15 @@ setup(AcquireRuntime* runtime, const char* filename) strlen(filename) + 1, nullptr, 0, - sample_spacing_um, + {1, 1}, 3)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 0, - SIZED("x") + 1, + SIZED("z") + 1, DimensionType_Space, - frame_width, - frame_width, + 0, + chunk_planes, 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 1, @@ -124,10 +122,10 @@ setup(AcquireRuntime* runtime, const char* filename) 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 2, - SIZED("z") + 1, + SIZED("x") + 1, DimensionType_Space, - 0, - chunk_planes, + frame_width, + frame_width, 0)); CHECK(storage_properties_set_enable_multiscale( @@ -236,8 +234,7 @@ validate() { CHECK(fs::is_directory(TEST ".zarr")); - const auto external_metadata_path = - fs::path(TEST ".zarr") / "0" / ".zattrs"; + const auto external_metadata_path = fs::path(TEST ".zarr") / "acquire.json"; CHECK(fs::is_regular_file(external_metadata_path)); CHECK(fs::file_size(external_metadata_path) == 2); // "{}" @@ -276,7 +273,7 @@ validate() const auto& scale = coord_trans["scale"]; ASSERT_EQ(float, "%f", std::pow(2.f, i), scale[0].get()); - ASSERT_EQ(float, "%f", 2.0f * std::pow(2.f, i), scale[1].get()); + ASSERT_EQ(float, "%f", std::pow(2.f, i), scale[1].get()); ASSERT_EQ(float, "%f", std::pow(2.f, i), scale[2].get()); } diff --git a/tests/driver/repeat-start.cpp b/tests/driver/repeat-start.cpp index e65eea9e..e54ace03 100644 --- a/tests/driver/repeat-start.cpp +++ b/tests/driver/repeat-start.cpp @@ -94,30 +94,30 @@ configure(AcquireRuntime* runtime) CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 0, - SIZED("x") + 1, - DimensionType_Space, - 64, + SIZED("t") + 1, + DimensionType_Time, + 0, 32, 1)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 1, - SIZED("y") + 1, - DimensionType_Space, - 48, - 32, - 1)); - CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 2, SIZED("c") + 1, DimensionType_Channel, 1, 1, 1)); + CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, + 2, + SIZED("y") + 1, + DimensionType_Space, + 48, + 32, + 1)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 3, - SIZED("t") + 1, - DimensionType_Time, - 0, + SIZED("x") + 1, + DimensionType_Space, + 64, 32, 1)); @@ -165,10 +165,6 @@ validate(AcquireRuntime* runtime) metadata_path = test_path / "meta" / "root.group.json"; CHECK(fs::is_regular_file(metadata_path)); - f = std::ifstream(metadata_path); - metadata = json::parse(f); - CHECK("" == metadata["attributes"]["acquire"]); - // check the array metadata file metadata_path = test_path / "meta" / "root" / "0.array.json"; CHECK(fs::is_regular_file(metadata_path)); diff --git a/tests/driver/restart-stopped-zarr-resets-threadpool.cpp b/tests/driver/restart-stopped-zarr-resets-threadpool.cpp index cdb7abfc..e9415032 100644 --- a/tests/driver/restart-stopped-zarr-resets-threadpool.cpp +++ b/tests/driver/restart-stopped-zarr-resets-threadpool.cpp @@ -95,11 +95,11 @@ configure(struct Storage* zarr) &props, 0, SIZED(TEST ".zarr"), nullptr, 0, { 0 }, 3); CHECK(storage_properties_set_dimension( - &props, 0, SIZED("x") + 1, DimensionType_Space, 64, 64, 0)); + &props, 2, SIZED("x") + 1, DimensionType_Space, 64, 64, 0)); CHECK(storage_properties_set_dimension( &props, 1, SIZED("y") + 1, DimensionType_Space, 48, 48, 0)); CHECK(storage_properties_set_dimension( - &props, 2, SIZED("t") + 1, DimensionType_Time, 0, 1, 0)); + &props, 0, SIZED("t") + 1, DimensionType_Time, 0, 1, 0)); CHECK(DeviceState_Armed == zarr->set(zarr, &props)); diff --git a/tests/driver/unit-tests.cpp b/tests/driver/unit-tests.cpp deleted file mode 100644 index e02880b8..00000000 --- a/tests/driver/unit-tests.cpp +++ /dev/null @@ -1,120 +0,0 @@ -// This is a "unit test" driver. -// -// Adding unit test functions here will run them as part of the CTest suite -// in a standardized fashion. -// -// Unit tests should be focused on testing the smallest logically isolated -// parts of the code. Practically, this means they should live close to the -// code they're testing. That is usually under the public interface -// defined by this module - if you're test uses a private interface that's a -// good sign it might be a unit test. -// -// Adding a new unit test: -// 1. Define your unit test in the same source file as what you're testing. -// 2. Add it to the declarations list below. See TEST DECLARATIONS. -// 3. Add it to the test list. See TEST LIST. -// -// Template: -// -// ```c -// #ifndef NO_UNIT_TESTS -// int -// unit_test__my_descriptive_test_name() -// { -// // do stuff -// return 1; // success -// Error: -// return 0; // failure -// } -// #endif // NO_UNIT_TESTS -// ``` - -#include "platform.h" -#include "logger.h" - -#include -#include -#include - -#define L (aq_logger) -#define LOG(...) L(0, __FILE__, __LINE__, __FUNCTION__, __VA_ARGS__) -#define ERR(...) L(1, __FILE__, __LINE__, __FUNCTION__, __VA_ARGS__) - -void -reporter(int is_error, - const char* file, - int line, - const char* function, - const char* msg) -{ - fprintf(is_error ? stderr : stdout, - "%s%s(%d) - %s: %s\n", - is_error ? "ERROR " : "", - file, - line, - function, - msg); -} - -typedef struct Driver* (*init_func_t)(void (*reporter)(int is_error, - const char* file, - int line, - const char* function, - const char* msg)); -// -// TEST DRIVER -// - -int -main() -{ - logger_set_reporter(reporter); - struct lib lib = { 0 }; - if (!lib_open_by_name(&lib, "acquire-driver-zarr")) { - ERR("Failed to open \"acquire-driver-zarr\"."); - exit(2); - } - - struct testcase - { - const char* name; - int (*test)(); - }; - const std::vector tests{ -#define CASE(e) { .name = #e, .test = (int (*)())lib_load(&lib, #e) } - CASE(unit_test__trim), - CASE(unit_test__split_uri), - CASE(unit_test__shard_index_for_chunk), - CASE(unit_test__shard_internal_index), - CASE(unit_test__average_frame), - CASE(unit_test__thread_pool__push_to_job_queue), - CASE(unit_test__s3_connection__put_object), - CASE(unit_test__s3_connection__upload_multipart_object), - CASE(unit_test__sink_creator__create_chunk_file_sinks), - CASE(unit_test__sink_creator__create_shard_file_sinks), - CASE(unit_test__chunk_lattice_index), - CASE(unit_test__tile_group_offset), - CASE(unit_test__chunk_internal_offset), - CASE(unit_test__writer__write_frame_to_chunks), - CASE(unit_test__downsample_writer_config), - CASE(unit_test__writer__write_frame_to_chunks), - CASE(unit_test__zarrv2_writer__write_even), - CASE(unit_test__zarrv2_writer__write_ragged_append_dim), - CASE(unit_test__zarrv2_writer__write_ragged_internal_dim), - CASE(unit_test__zarrv3_writer__write_even), - CASE(unit_test__zarrv3_writer__write_ragged_append_dim), - CASE(unit_test__zarrv3_writer__write_ragged_internal_dim), -#undef CASE - }; - - bool any = false; - for (const auto& test : tests) { - LOG("Running %s", test.name); - if (!(test.test())) { - ERR("unit test failed: %s", test.name); - any = true; - } - } - lib_close(&lib); - return any; -} diff --git a/tests/driver/write-zarr-v2-compressed-multiscale.cpp b/tests/driver/write-zarr-v2-compressed-multiscale.cpp index 1e4ed823..8cbd1b12 100644 --- a/tests/driver/write-zarr-v2-compressed-multiscale.cpp +++ b/tests/driver/write-zarr-v2-compressed-multiscale.cpp @@ -128,31 +128,31 @@ acquire(AcquireRuntime* runtime, const char* filename) CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 0, - SIZED("x") + 1, - DimensionType_Space, - frame_width, - chunk_width, + SIZED("t") + 1, + DimensionType_Time, + 0, + chunk_planes, 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 1, - SIZED("y") + 1, - DimensionType_Space, - frame_height, - chunk_height, - 0)); - CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 2, SIZED("c") + 1, DimensionType_Channel, 1, 1, 0)); + CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, + 2, + SIZED("y") + 1, + DimensionType_Space, + frame_height, + chunk_height, + 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 3, - SIZED("t") + 1, - DimensionType_Time, - 0, - chunk_planes, + SIZED("x") + 1, + DimensionType_Space, + frame_width, + chunk_width, 0)); CHECK(storage_properties_set_enable_multiscale( @@ -262,8 +262,7 @@ validate() { CHECK(fs::is_directory(TEST ".zarr")); - const auto external_metadata_path = - fs::path(TEST ".zarr") / "0" / ".zattrs"; + const auto external_metadata_path = fs::path(TEST ".zarr") / "acquire.json"; CHECK(fs::is_regular_file(external_metadata_path)); CHECK(fs::file_size(external_metadata_path) > 0); diff --git a/tests/driver/write-zarr-v2-compressed-with-chunking-and-rollover.cpp b/tests/driver/write-zarr-v2-compressed-with-chunking-and-rollover.cpp index 411b1eb0..216bc926 100644 --- a/tests/driver/write-zarr-v2-compressed-with-chunking-and-rollover.cpp +++ b/tests/driver/write-zarr-v2-compressed-with-chunking-and-rollover.cpp @@ -109,31 +109,31 @@ acquire(AcquireRuntime* runtime, const char* filename) CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 0, - SIZED("x") + 1, - DimensionType_Space, - frame_width, - chunk_width, + SIZED("t") + 1, + DimensionType_Time, + 0, + chunk_planes, 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 1, - SIZED("y") + 1, - DimensionType_Space, - frame_height, - chunk_height, - 0)); - CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 2, SIZED("c") + 1, DimensionType_Channel, 1, 1, 0)); + CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, + 2, + SIZED("y") + 1, + DimensionType_Space, + frame_height, + chunk_height, + 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 3, - SIZED("t") + 1, - DimensionType_Time, - 0, - chunk_planes, + SIZED("x") + 1, + DimensionType_Space, + frame_width, + chunk_width, 0)); props.video[0].camera.settings.binning = 1; @@ -157,8 +157,7 @@ validate() { CHECK(fs::is_directory(TEST ".zarr")); - const auto external_metadata_path = - fs::path(TEST ".zarr") / "0" / ".zattrs"; + const auto external_metadata_path = fs::path(TEST ".zarr") / "acquire.json"; CHECK(fs::is_regular_file(external_metadata_path)); CHECK(fs::file_size(external_metadata_path) > 0); diff --git a/tests/driver/write-zarr-v2-compressed-with-chunking.cpp b/tests/driver/write-zarr-v2-compressed-with-chunking.cpp index 8944de71..51f4eca5 100644 --- a/tests/driver/write-zarr-v2-compressed-with-chunking.cpp +++ b/tests/driver/write-zarr-v2-compressed-with-chunking.cpp @@ -107,31 +107,31 @@ acquire(AcquireRuntime* runtime, const char* filename) CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 0, - SIZED("x") + 1, - DimensionType_Space, - frame_width, - chunk_width, + SIZED("t") + 1, + DimensionType_Time, + 0, + chunk_planes, 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 1, - SIZED("y") + 1, - DimensionType_Space, - frame_height, - chunk_height, - 0)); - CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 2, SIZED("c") + 1, DimensionType_Channel, 1, 1, 0)); + CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, + 2, + SIZED("y") + 1, + DimensionType_Space, + frame_height, + chunk_height, + 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 3, - SIZED("t") + 1, - DimensionType_Time, - 0, - chunk_planes, + SIZED("x") + 1, + DimensionType_Space, + frame_width, + chunk_width, 0)); props.video[0].camera.settings.binning = 1; @@ -154,8 +154,7 @@ validate() { CHECK(fs::is_directory(TEST ".zarr")); - const auto external_metadata_path = - fs::path(TEST ".zarr") / "0" / ".zattrs"; + const auto external_metadata_path = fs::path(TEST ".zarr") / "acquire.json"; CHECK(fs::is_regular_file(external_metadata_path)); CHECK(fs::file_size(external_metadata_path) > 0); diff --git a/tests/driver/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp b/tests/driver/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp index 0e732a49..483e539f 100644 --- a/tests/driver/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp +++ b/tests/driver/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp @@ -104,31 +104,31 @@ acquire(AcquireRuntime* runtime, const char* filename) CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 0, - SIZED("x") + 1, - DimensionType_Space, - frame_width, - chunk_width, + SIZED("t") + 1, + DimensionType_Time, + 0, + frames_per_chunk, 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 1, - SIZED("y") + 1, - DimensionType_Space, - frame_height, - chunk_height, - 0)); - CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 2, SIZED("c") + 1, DimensionType_Channel, 1, 1, 0)); + CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, + 2, + SIZED("y") + 1, + DimensionType_Space, + frame_height, + chunk_height, + 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 3, - SIZED("t") + 1, - DimensionType_Time, - 0, - frames_per_chunk, + SIZED("x") + 1, + DimensionType_Space, + frame_width, + chunk_width, 0)); props.video[0].camera.settings.binning = 1; @@ -216,8 +216,7 @@ validate() { CHECK(fs::is_directory(TEST ".zarr")); - const auto external_metadata_path = - fs::path(TEST ".zarr") / "0" / ".zattrs"; + const auto external_metadata_path = fs::path(TEST ".zarr") / "acquire.json"; CHECK(fs::is_regular_file(external_metadata_path)); ASSERT_GT(int, "%d", fs::file_size(external_metadata_path), 0); diff --git a/tests/driver/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp b/tests/driver/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp index 6a1b6c1f..147c3ec5 100644 --- a/tests/driver/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp +++ b/tests/driver/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp @@ -111,31 +111,31 @@ acquire(AcquireRuntime* runtime, const char* filename) CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 0, - SIZED("x") + 1, - DimensionType_Space, - frame_width, - frame_width, + SIZED("t") + 1, + DimensionType_Time, + 0, + chunk_planes, 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 1, - SIZED("y") + 1, - DimensionType_Space, - frame_height, - frame_height, - 0)); - CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 2, SIZED("c") + 1, DimensionType_Channel, 1, 1, 0)); + CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, + 2, + SIZED("y") + 1, + DimensionType_Space, + frame_height, + frame_height, + 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 3, - SIZED("t") + 1, - DimensionType_Time, - 0, - chunk_planes, + SIZED("x") + 1, + DimensionType_Space, + frame_width, + frame_width, 0)); CHECK(storage_properties_set_enable_multiscale( @@ -245,8 +245,7 @@ validate() { CHECK(fs::is_directory(TEST ".zarr")); - const auto external_metadata_path = - fs::path(TEST ".zarr") / "0" / ".zattrs"; + const auto external_metadata_path = fs::path(TEST ".zarr") / "acquire.json"; CHECK(fs::is_regular_file(external_metadata_path)); CHECK(fs::file_size(external_metadata_path) > 0); diff --git a/tests/driver/write-zarr-v2-raw-multiscale.cpp b/tests/driver/write-zarr-v2-raw-multiscale.cpp index 88a8e5b8..4dddc279 100644 --- a/tests/driver/write-zarr-v2-raw-multiscale.cpp +++ b/tests/driver/write-zarr-v2-raw-multiscale.cpp @@ -118,31 +118,31 @@ acquire(AcquireRuntime* runtime, const char* filename) CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 0, - SIZED("x") + 1, - DimensionType_Space, - frame_width, - chunk_width, + SIZED("t") + 1, + DimensionType_Time, + 0, + chunk_planes, 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 1, - SIZED("y") + 1, - DimensionType_Space, - frame_height, - chunk_height, - 0)); - CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 2, SIZED("c") + 1, DimensionType_Channel, 1, 1, 0)); + CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, + 2, + SIZED("y") + 1, + DimensionType_Space, + frame_height, + chunk_height, + 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 3, - SIZED("t") + 1, - DimensionType_Time, - 0, - chunk_planes, + SIZED("x") + 1, + DimensionType_Space, + frame_width, + chunk_width, 0)); CHECK(storage_properties_set_enable_multiscale( @@ -252,8 +252,7 @@ validate() { CHECK(fs::is_directory(TEST ".zarr")); - const auto external_metadata_path = - fs::path(TEST ".zarr") / "0" / ".zattrs"; + const auto external_metadata_path = fs::path(TEST ".zarr") / "acquire.json"; CHECK(fs::is_regular_file(external_metadata_path)); CHECK(fs::file_size(external_metadata_path) > 0); diff --git a/tests/driver/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp b/tests/driver/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp index 18379436..c4675d51 100644 --- a/tests/driver/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp +++ b/tests/driver/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp @@ -97,31 +97,31 @@ acquire(AcquireRuntime* runtime, const char* filename) CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 0, - SIZED("x") + 1, - DimensionType_Space, - frame_width, - chunk_width, + SIZED("t") + 1, + DimensionType_Time, + 0, + chunk_planes, 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 1, - SIZED("y") + 1, - DimensionType_Space, - frame_height, - chunk_height, - 0)); - CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 2, SIZED("c") + 1, DimensionType_Channel, 1, 1, 0)); + CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, + 2, + SIZED("y") + 1, + DimensionType_Space, + frame_height, + chunk_height, + 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 3, - SIZED("t") + 1, - DimensionType_Time, - 0, - chunk_planes, + SIZED("x") + 1, + DimensionType_Space, + frame_width, + chunk_width, 0)); props.video[0].camera.settings.binning = 1; @@ -145,8 +145,7 @@ validate() { CHECK(fs::is_directory(TEST ".zarr")); - const auto external_metadata_path = - fs::path(TEST ".zarr") / "0" / ".zattrs"; + const auto external_metadata_path = fs::path(TEST ".zarr") / "acquire.json"; CHECK(fs::is_regular_file(external_metadata_path)); CHECK(fs::file_size(external_metadata_path) > 0); diff --git a/tests/driver/write-zarr-v2-raw-with-even-chunking.cpp b/tests/driver/write-zarr-v2-raw-with-even-chunking.cpp index 58fbdb9b..429674db 100644 --- a/tests/driver/write-zarr-v2-raw-with-even-chunking.cpp +++ b/tests/driver/write-zarr-v2-raw-with-even-chunking.cpp @@ -97,31 +97,31 @@ acquire(AcquireRuntime* runtime, const char* filename) CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 0, - SIZED("x") + 1, - DimensionType_Space, - frame_width, - chunk_width, + SIZED("t") + 1, + DimensionType_Time, + 0, + chunk_planes, 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 1, - SIZED("y") + 1, - DimensionType_Space, - frame_height, - chunk_height, - 0)); - CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 2, SIZED("c") + 1, DimensionType_Channel, 1, 1, 0)); + CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, + 2, + SIZED("y") + 1, + DimensionType_Space, + frame_height, + chunk_height, + 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 3, - SIZED("t") + 1, - DimensionType_Time, - 0, - chunk_planes, + SIZED("x") + 1, + DimensionType_Space, + frame_width, + chunk_width, 0)); props.video[0].camera.settings.binning = 1; @@ -144,8 +144,7 @@ validate() { CHECK(fs::is_directory(TEST ".zarr")); - const auto external_metadata_path = - fs::path(TEST ".zarr") / "0" / ".zattrs"; + const auto external_metadata_path = fs::path(TEST ".zarr") / "acquire.json"; CHECK(fs::is_regular_file(external_metadata_path)); CHECK(fs::file_size(external_metadata_path) > 0); diff --git a/tests/driver/write-zarr-v2-raw-with-ragged-chunking.cpp b/tests/driver/write-zarr-v2-raw-with-ragged-chunking.cpp index 971eb66e..4fb7eda1 100644 --- a/tests/driver/write-zarr-v2-raw-with-ragged-chunking.cpp +++ b/tests/driver/write-zarr-v2-raw-with-ragged-chunking.cpp @@ -111,31 +111,31 @@ acquire(AcquireRuntime* runtime, const char* filename) CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 0, - SIZED("x") + 1, - DimensionType_Space, - frame_width, - chunk_width, + SIZED("t") + 1, + DimensionType_Time, + 0, + chunk_planes, 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 1, - SIZED("y") + 1, - DimensionType_Space, - frame_height, - chunk_height, - 0)); - CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 2, SIZED("c") + 1, DimensionType_Channel, 1, 1, 0)); + CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, + 2, + SIZED("y") + 1, + DimensionType_Space, + frame_height, + chunk_height, + 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 3, - SIZED("t") + 1, - DimensionType_Time, - 0, - chunk_planes, + SIZED("x") + 1, + DimensionType_Space, + frame_width, + chunk_width, 0)); props.video[0].camera.settings.binning = 1; @@ -220,8 +220,7 @@ validate() { CHECK(fs::is_directory(TEST ".zarr")); - const auto external_metadata_path = - fs::path(TEST ".zarr") / "0" / ".zattrs"; + const auto external_metadata_path = fs::path(TEST ".zarr") / "acquire.json"; CHECK(fs::is_regular_file(external_metadata_path)); ASSERT_GT(size_t, "%zu", fs::file_size(external_metadata_path), 0); diff --git a/tests/driver/write-zarr-v2-raw.cpp b/tests/driver/write-zarr-v2-raw.cpp index d213ecad..32da89f2 100644 --- a/tests/driver/write-zarr-v2-raw.cpp +++ b/tests/driver/write-zarr-v2-raw.cpp @@ -105,31 +105,31 @@ acquire(AcquireRuntime* runtime, const char* filename) CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 0, - SIZED("x") + 1, - DimensionType_Space, - frame_width, - frame_width, + SIZED("t") + 1, + DimensionType_Time, + 0, + frames_per_chunk, 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 1, - SIZED("y") + 1, - DimensionType_Space, - frame_height, - frame_height, - 0)); - CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 2, SIZED("c") + 1, DimensionType_Channel, 1, 1, 0)); + CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, + 2, + SIZED("y") + 1, + DimensionType_Space, + frame_height, + frame_height, + 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 3, - SIZED("t") + 1, - DimensionType_Time, - 0, - frames_per_chunk, + SIZED("x") + 1, + DimensionType_Space, + frame_width, + frame_width, 0)); props.video[0].camera.settings.binning = 1; @@ -217,8 +217,7 @@ validate() { CHECK(fs::is_directory(TEST ".zarr")); - const auto external_metadata_path = - fs::path(TEST ".zarr") / "0" / ".zattrs"; + const auto external_metadata_path = fs::path(TEST ".zarr") / "acquire.json"; CHECK(fs::is_regular_file(external_metadata_path)); ASSERT_GT(int, "%d", fs::file_size(external_metadata_path), 0); diff --git a/tests/driver/write-zarr-v2-to-s3.cpp b/tests/driver/write-zarr-v2-to-s3.cpp index 4de38fae..b5505106 100644 --- a/tests/driver/write-zarr-v2-to-s3.cpp +++ b/tests/driver/write-zarr-v2-to-s3.cpp @@ -251,13 +251,13 @@ configure(AcquireRuntime* runtime) CHECK(bucket_exists(client)); } - std::string uri = s3_endpoint + ("/" + s3_bucket_name); + std::string uri = s3_endpoint + ("/" + s3_bucket_name) + ("/" TEST); storage_properties_init(&props.video[0].storage.settings, 0, uri.c_str(), uri.length() + 1, - nullptr, - 0, + R"({"hello":"world"})", + sizeof(R"({"hello":"world"})"), {}, 3); CHECK(storage_properties_set_access_key_and_secret( @@ -270,10 +270,10 @@ configure(AcquireRuntime* runtime) CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 0, - SIZED("x") + 1, - DimensionType_Space, - 1920, - 1920, + SIZED("t") + 1, + DimensionType_Time, + 0, + 5, 1)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 1, @@ -284,10 +284,10 @@ configure(AcquireRuntime* runtime) 2)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 2, - SIZED("t") + 1, - DimensionType_Time, - 0, - 5, + SIZED("x") + 1, + DimensionType_Space, + 1920, + 1920, 1)); OK(acquire_configure(runtime, &props)); @@ -304,14 +304,14 @@ void validate_and_cleanup(AcquireRuntime* runtime) { std::vector paths{ - ".zgroup", - ".zattrs", - "0/.zarray", - "0/.zattrs", + (TEST "/.zgroup"), + (TEST "/.zattrs"), + (TEST "/0/.zarray"), + (TEST "/acquire.json"), }; for (auto i = 0; i < 20; ++i) { - paths.push_back("0/" + std::to_string(i) + "/0/0"); - paths.push_back("0/" + std::to_string(i) + "/1/0"); + paths.push_back((TEST "/0/") + std::to_string(i) + "/0/0"); + paths.push_back((TEST "/0/") + std::to_string(i) + "/1/0"); } minio::s3::BaseUrl url(s3_endpoint); @@ -325,7 +325,10 @@ validate_and_cleanup(AcquireRuntime* runtime) try { for (const auto& path : paths) { - CHECK(object_exists(client, path)); + EXPECT(object_exists(client, path), + "Object %s does not exist in bucket %s", + path.c_str(), + s3_bucket_name.c_str()); } } catch (const std::exception& e) { ERR("Exception: %s", e.what()); diff --git a/tests/driver/write-zarr-v2-with-lz4-compression.cpp b/tests/driver/write-zarr-v2-with-lz4-compression.cpp index 686d0419..b5df3cd4 100644 --- a/tests/driver/write-zarr-v2-with-lz4-compression.cpp +++ b/tests/driver/write-zarr-v2-with-lz4-compression.cpp @@ -105,31 +105,31 @@ acquire(AcquireRuntime* runtime, const char* filename) CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 0, - SIZED("x") + 1, - DimensionType_Space, - frame_width, - frame_width, + SIZED("t") + 1, + DimensionType_Time, + 0, + frames_per_chunk, 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 1, - SIZED("y") + 1, - DimensionType_Space, - frame_height, - frame_height, - 0)); - CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 2, SIZED("c") + 1, DimensionType_Channel, 1, 1, 0)); + CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, + 2, + SIZED("y") + 1, + DimensionType_Space, + frame_height, + frame_height, + 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 3, - SIZED("t") + 1, - DimensionType_Time, - 0, - frames_per_chunk, + SIZED("x") + 1, + DimensionType_Space, + frame_width, + frame_width, 0)); props.video[0].camera.settings.binning = 1; @@ -152,8 +152,7 @@ validate() { CHECK(fs::is_directory(TEST ".zarr")); - const auto external_metadata_path = - fs::path(TEST ".zarr") / "0" / ".zattrs"; + const auto external_metadata_path = fs::path(TEST ".zarr") / "acquire.json"; CHECK(fs::is_regular_file(external_metadata_path)); ASSERT_GT(int, "%d", fs::file_size(external_metadata_path), 0); diff --git a/tests/driver/write-zarr-v2-with-zstd-compression.cpp b/tests/driver/write-zarr-v2-with-zstd-compression.cpp index 5430d558..77ed12ce 100644 --- a/tests/driver/write-zarr-v2-with-zstd-compression.cpp +++ b/tests/driver/write-zarr-v2-with-zstd-compression.cpp @@ -105,31 +105,31 @@ acquire(AcquireRuntime* runtime, const char* filename) CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 0, - SIZED("x") + 1, - DimensionType_Space, - frame_width, - frame_width, + SIZED("t") + 1, + DimensionType_Time, + 0, + frames_per_chunk, 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 1, - SIZED("y") + 1, - DimensionType_Space, - frame_height, - frame_height, - 0)); - CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 2, SIZED("c") + 1, DimensionType_Channel, 1, 1, 0)); + CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, + 2, + SIZED("y") + 1, + DimensionType_Space, + frame_height, + frame_height, + 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 3, - SIZED("t") + 1, - DimensionType_Time, - 0, - frames_per_chunk, + SIZED("x") + 1, + DimensionType_Space, + frame_width, + frame_width, 0)); props.video[0].camera.settings.binning = 1; @@ -152,8 +152,7 @@ validate() { CHECK(fs::is_directory(TEST ".zarr")); - const auto external_metadata_path = - fs::path(TEST ".zarr") / "0" / ".zattrs"; + const auto external_metadata_path = fs::path(TEST ".zarr") / "acquire.json"; CHECK(fs::is_regular_file(external_metadata_path)); ASSERT_GT(int, "%d", fs::file_size(external_metadata_path), 0); diff --git a/tests/driver/write-zarr-v3-compressed.cpp b/tests/driver/write-zarr-v3-compressed.cpp index 734e2521..f6672ccd 100644 --- a/tests/driver/write-zarr-v3-compressed.cpp +++ b/tests/driver/write-zarr-v3-compressed.cpp @@ -108,34 +108,34 @@ setup(AcquireRuntime* runtime) 0, (char*)filename, strlen(filename) + 1, - nullptr, - 0, + R"({"hello":"world"})", + sizeof(R"({"hello":"world"})"), sample_spacing_um, 4)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 0, + 3, SIZED("x") + 1, DimensionType_Space, frame_width, chunk_width, shard_width)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 1, + 2, SIZED("y") + 1, DimensionType_Space, frame_height, chunk_height, shard_height)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 2, + 1, SIZED("c") + 1, DimensionType_Channel, 1, 1, 1)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 3, + 0, SIZED("t") + 1, DimensionType_Time, 0, @@ -248,9 +248,13 @@ validate() metadata_path = test_path / "meta" / "root.group.json"; CHECK(fs::is_regular_file(metadata_path)); + // check the external metadata file + metadata_path = test_path / "meta" / "acquire.json"; + CHECK(fs::is_regular_file(metadata_path)); + f = std::ifstream(metadata_path); metadata = json::parse(f); - CHECK("" == metadata["attributes"]["acquire"]); + CHECK("world" == metadata["hello"]); // check the array metadata file metadata_path = test_path / "meta" / "root" / "0.array.json"; diff --git a/tests/driver/write-zarr-v3-raw-chunk-exceeds-array.cpp b/tests/driver/write-zarr-v3-raw-chunk-exceeds-array.cpp index cfc3d7b0..e3536a2c 100644 --- a/tests/driver/write-zarr-v3-raw-chunk-exceeds-array.cpp +++ b/tests/driver/write-zarr-v3-raw-chunk-exceeds-array.cpp @@ -114,11 +114,11 @@ setup(AcquireRuntime* runtime) CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 0, - SIZED("x") + 1, - DimensionType_Space, - frame_width, - chunk_width, - shard_width)); + SIZED("t") + 1, + DimensionType_Time, + 0, + frames_per_chunk, + 1)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 1, SIZED("y") + 1, @@ -128,11 +128,11 @@ setup(AcquireRuntime* runtime) shard_height)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 2, - SIZED("t") + 1, - DimensionType_Time, - 0, - frames_per_chunk, - 1)); + SIZED("x") + 1, + DimensionType_Space, + frame_width, + chunk_width, + shard_width)); props.video[0].camera.settings.binning = 1; props.video[0].camera.settings.pixel_type = SampleType_u8; @@ -240,9 +240,13 @@ validate() metadata_path = test_path / "meta" / "root.group.json"; CHECK(fs::is_regular_file(metadata_path)); + // check the external metadata file + metadata_path = test_path / "meta" / "acquire.json"; + CHECK(fs::is_regular_file(metadata_path)); + f = std::ifstream(metadata_path); metadata = json::parse(f); - CHECK("" == metadata["attributes"]["acquire"]); + CHECK(metadata.empty()); // check the array metadata file metadata_path = test_path / "meta" / "root" / "0.array.json"; diff --git a/tests/driver/write-zarr-v3-raw-multiscale.cpp b/tests/driver/write-zarr-v3-raw-multiscale.cpp index b5057753..61ace086 100644 --- a/tests/driver/write-zarr-v3-raw-multiscale.cpp +++ b/tests/driver/write-zarr-v3-raw-multiscale.cpp @@ -132,31 +132,31 @@ configure(AcquireRuntime* runtime) // configure storage dimensions CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 0, - SIZED("x") + 1, - DimensionType_Space, - frame_width, - chunk_width, + SIZED("t") + 1, + DimensionType_Time, + 0, + chunk_planes, 1)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 1, - SIZED("y") + 1, - DimensionType_Space, - frame_height, - chunk_height, - 1)); - CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 2, SIZED("c") + 1, DimensionType_Channel, 1, 1, 1)); + CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, + 2, + SIZED("y") + 1, + DimensionType_Space, + frame_height, + chunk_height, + 1)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 3, - SIZED("t") + 1, - DimensionType_Time, - 0, - chunk_planes, + SIZED("x") + 1, + DimensionType_Space, + frame_width, + chunk_width, 1)); // configure acquisition diff --git a/tests/driver/write-zarr-v3-raw-with-ragged-sharding.cpp b/tests/driver/write-zarr-v3-raw-with-ragged-sharding.cpp index b3ccd51a..b35574e8 100644 --- a/tests/driver/write-zarr-v3-raw-with-ragged-sharding.cpp +++ b/tests/driver/write-zarr-v3-raw-with-ragged-sharding.cpp @@ -113,11 +113,11 @@ setup(AcquireRuntime* runtime) CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 0, - SIZED("x") + 1, - DimensionType_Space, - frame_width, - chunk_width, - shard_width)); + SIZED("t") + 1, + DimensionType_Time, + 0, + frames_per_chunk, + 1)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 1, SIZED("y") + 1, @@ -127,11 +127,11 @@ setup(AcquireRuntime* runtime) shard_height)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 2, - SIZED("t") + 1, - DimensionType_Time, - 0, - frames_per_chunk, - 1)); + SIZED("x") + 1, + DimensionType_Space, + frame_width, + chunk_width, + shard_width)); props.video[0].camera.settings.binning = 1; props.video[0].camera.settings.pixel_type = SampleType_u8; @@ -239,9 +239,13 @@ validate() metadata_path = test_path / "meta" / "root.group.json"; CHECK(fs::is_regular_file(metadata_path)); + // check the external metadata file + metadata_path = test_path / "meta" / "acquire.json"; + CHECK(fs::is_regular_file(metadata_path)); + f = std::ifstream(metadata_path); metadata = json::parse(f); - CHECK("" == metadata["attributes"]["acquire"]); + CHECK(metadata.empty()); // check the array metadata file metadata_path = test_path / "meta" / "root" / "0.array.json"; @@ -307,10 +311,10 @@ validate() for (auto i = 0; i < indices.size(); i += 2) { ASSERT_EQ( - uint64_t, "%llu", (i / 2) * bytes_per_chunk, indices.at(i)); + uint64_t, "%zu", (i / 2) * bytes_per_chunk, indices.at(i)); } for (auto i = 1; i < indices.size(); i += 2) { - ASSERT_EQ(uint64_t, "%llu", bytes_per_chunk, indices.at(i)); + ASSERT_EQ(uint64_t, "%zu", bytes_per_chunk, indices.at(i)); } } @@ -336,16 +340,16 @@ validate() size_t offset = 0; for (auto i = 0; i < indices.size(); i += 12) { - ASSERT_EQ(uint64_t, "%llu", offset, indices.at(i)); - ASSERT_EQ(uint64_t, "%llu", bytes_per_chunk, indices.at(i + 1)); + ASSERT_EQ(uint64_t, "%zu", offset, indices.at(i)); + ASSERT_EQ(uint64_t, "%zu", bytes_per_chunk, indices.at(i + 1)); ASSERT_EQ( - uint64_t, "%llu", offset + bytes_per_chunk, indices.at(i + 2)); - ASSERT_EQ(uint64_t, "%llu", bytes_per_chunk, indices.at(i + 3)); + uint64_t, "%zu", offset + bytes_per_chunk, indices.at(i + 2)); + ASSERT_EQ(uint64_t, "%zu", bytes_per_chunk, indices.at(i + 3)); // the rest of the indices should be empty for (auto j = 4; j < 12; ++j) { ASSERT_EQ(uint64_t, - "%llu", + "%zu", std::numeric_limits::max(), indices.at(i + j)); } @@ -376,14 +380,14 @@ validate() for (auto i = 0; i < 12; i += 2) { ASSERT_EQ( - uint64_t, "%llu", (i / 2) * bytes_per_chunk, indices.at(i)); - ASSERT_EQ(uint64_t, "%llu", bytes_per_chunk, indices.at(i + 1)); + uint64_t, "%zu", (i / 2) * bytes_per_chunk, indices.at(i)); + ASSERT_EQ(uint64_t, "%zu", bytes_per_chunk, indices.at(i + 1)); } // the rest of the indices should be empty for (auto i = 12; i < indices.size(); ++i) { ASSERT_EQ(uint64_t, - "%llu", + "%zu", std::numeric_limits::max(), indices.at(i)); } @@ -409,14 +413,14 @@ validate() std::vector indices(2 * chunks_per_full_shard); shard_file.read(reinterpret_cast(indices.data()), index_size); - ASSERT_EQ(uint64_t, "%llu", 0, indices.at(0)); - ASSERT_EQ(uint64_t, "%llu", bytes_per_chunk, indices.at(1)); - ASSERT_EQ(uint64_t, "%llu", bytes_per_chunk, indices.at(2)); - ASSERT_EQ(uint64_t, "%llu", bytes_per_chunk, indices.at(3)); + ASSERT_EQ(uint64_t, "%zu", 0, indices.at(0)); + ASSERT_EQ(uint64_t, "%zu", bytes_per_chunk, indices.at(1)); + ASSERT_EQ(uint64_t, "%zu", bytes_per_chunk, indices.at(2)); + ASSERT_EQ(uint64_t, "%zu", bytes_per_chunk, indices.at(3)); for (auto i = 4; i < indices.size(); ++i) { ASSERT_EQ(uint64_t, - "%llu", + "%zu", std::numeric_limits::max(), indices.at(i)); } diff --git a/tests/driver/write-zarr-v3-raw.cpp b/tests/driver/write-zarr-v3-raw.cpp index 64d84197..5b90a154 100644 --- a/tests/driver/write-zarr-v3-raw.cpp +++ b/tests/driver/write-zarr-v3-raw.cpp @@ -115,32 +115,32 @@ setup(AcquireRuntime* runtime) CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 0, - SIZED("x") + 1, - DimensionType_Space, - frame_width, - chunk_width, - shard_width)); + SIZED("t") + 1, + DimensionType_Time, + 0, + frames_per_chunk, + 1)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 1, - SIZED("y") + 1, - DimensionType_Space, - frame_height, - chunk_height, - shard_height)); - CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 2, SIZED("c") + 1, DimensionType_Channel, 1, 1, 1)); + CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, + 2, + SIZED("y") + 1, + DimensionType_Space, + frame_height, + chunk_height, + shard_height)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 3, - SIZED("t") + 1, - DimensionType_Time, - 0, - frames_per_chunk, - 1)); + SIZED("x") + 1, + DimensionType_Space, + frame_width, + chunk_width, + shard_width)); props.video[0].camera.settings.binning = 1; props.video[0].camera.settings.pixel_type = SampleType_u8; @@ -248,9 +248,13 @@ validate() metadata_path = test_path / "meta" / "root.group.json"; CHECK(fs::is_regular_file(metadata_path)); + // check the external metadata file + metadata_path = test_path / "meta" / "acquire.json"; + CHECK(fs::is_regular_file(metadata_path)); + f = std::ifstream(metadata_path); metadata = json::parse(f); - CHECK("" == metadata["attributes"]["acquire"]); + CHECK(metadata.empty()); // check the array metadata file metadata_path = test_path / "meta" / "root" / "0.array.json"; diff --git a/tests/driver/write-zarr-v3-to-s3.cpp b/tests/driver/write-zarr-v3-to-s3.cpp index 3474c053..c23483ad 100644 --- a/tests/driver/write-zarr-v3-to-s3.cpp +++ b/tests/driver/write-zarr-v3-to-s3.cpp @@ -251,13 +251,13 @@ configure(AcquireRuntime* runtime) CHECK(bucket_exists(client)); } - std::string uri = s3_endpoint + ("/" + s3_bucket_name); + std::string uri = s3_endpoint + ("/" + s3_bucket_name) + ("/" TEST); storage_properties_init(&props.video[0].storage.settings, 0, uri.c_str(), uri.length() + 1, - nullptr, - 0, + R"({"hello":"world"})", + sizeof(R"({"hello":"world"})"), {}, 3); CHECK(storage_properties_set_access_key_and_secret( @@ -270,10 +270,10 @@ configure(AcquireRuntime* runtime) CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 0, - SIZED("x") + 1, - DimensionType_Space, - 1920, - 1920, + SIZED("t") + 1, + DimensionType_Time, + 0, + 5, 1)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 1, @@ -284,10 +284,10 @@ configure(AcquireRuntime* runtime) 2)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, 2, - SIZED("t") + 1, - DimensionType_Time, - 0, - 5, + SIZED("x") + 1, + DimensionType_Space, + 1920, + 1920, 1)); OK(acquire_configure(runtime, &props)); @@ -305,11 +305,12 @@ validate_and_cleanup(AcquireRuntime* runtime) { CHECK(runtime); - std::vector paths{ "zarr.json", - "meta/root.group.json", - "meta/root/0.array.json" }; + std::vector paths{ (TEST "/zarr.json"), + (TEST "/meta/root.group.json"), + (TEST "/meta/root/0.array.json"), + (TEST "/meta/acquire.json") }; for (auto i = 0; i < 20; ++i) { - paths.push_back("data/root/0/c" + std::to_string(i) + "/0/0"); + paths.push_back((TEST "/data/root/0/c") + std::to_string(i) + "/0/0"); } minio::s3::BaseUrl url(s3_endpoint);