Skip to content

Commit 597f3b7

Browse files
author
birydrad
committedFeb 25, 2025
celldb: add test for load nonexisting cell, test thread safeness of CellUsageTree, fixes
1 parent c863c42 commit 597f3b7

7 files changed

+130
-61
lines changed
 

‎crypto/test/test-db.cpp

+34-1
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
#include <set>
5151
#include <map>
5252
#include <thread>
53+
#include <barrier>
5354

5455
#include <openssl/sha.h>
5556

@@ -1312,7 +1313,8 @@ void with_all_boc_options(F &&f, size_t tests_n, bool single_thread = false) {
13121313
// V2 - one thread
13131314
run({.async_executor = executor,
13141315
.kv_options = kv_options,
1315-
.options = DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 0, .executor = executor, .cache_ttl_max = 5}});
1316+
.options =
1317+
DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 0, .executor = executor, .cache_ttl_max = 5}});
13161318

13171319
// InMemory
13181320
for (auto use_arena : {false, true}) {
@@ -1353,6 +1355,8 @@ DynamicBagOfCellsDb::Stats test_dynamic_boc(BocOptions options) {
13531355
if (rnd() % 10 == 0) {
13541356
reload_db();
13551357
}
1358+
db.dboc->load_cell(vm::CellHash{}.as_slice()).ensure_error();
1359+
13561360
db.reset_loader();
13571361
Ref<Cell> old_root;
13581362
if (!old_root_hash.empty()) {
@@ -2901,6 +2905,35 @@ TEST(TonDb, BocRespectsUsageCell) {
29012905
ASSERT_STREQ(serialization, serialization_of_virtualized_cell);
29022906
}
29032907

2908+
TEST(UsageTree, ThreadSafe) {
2909+
size_t test_n = 100;
2910+
td::Random::Xorshift128plus rnd(123);
2911+
for (size_t test_i = 0; test_i < test_n; test_i++) {
2912+
auto cell = vm::gen_random_cell(rnd.fast(2, 100), rnd, false);
2913+
auto usage_tree = std::make_shared<vm::CellUsageTree>();
2914+
auto usage_cell = vm::UsageCell::create(cell, usage_tree->root_ptr());
2915+
std::ptrdiff_t threads_n = 1; // TODO: when CellUsageTree is thread safe, change it to 4
2916+
auto barrier = std::barrier{threads_n};
2917+
std::vector<std::thread> threads;
2918+
std::vector<vm::CellExplorer::Exploration> explorations(threads_n);
2919+
for (std::ptrdiff_t i = 0; i < threads_n; i++) {
2920+
threads.emplace_back([&, i = i]() {
2921+
barrier.arrive_and_wait();
2922+
explorations[i] = vm::CellExplorer::random_explore(usage_cell, rnd);
2923+
});
2924+
}
2925+
for (auto &thread : threads) {
2926+
thread.join();
2927+
}
2928+
auto proof = vm::MerkleProof::generate(cell, usage_tree.get());
2929+
auto virtualized_proof = vm::MerkleProof::virtualize(proof, 1);
2930+
for (auto &exploration : explorations) {
2931+
auto new_exploration = vm::CellExplorer::explore(virtualized_proof, exploration.ops);
2932+
ASSERT_EQ(exploration.log, new_exploration.log);
2933+
}
2934+
}
2935+
}
2936+
29042937
/*
29052938
vm::DynamicBagOfCellsDb::Stats test_dynamic_boc_respects_usage_cell(vm::BocOptions options) {
29062939
td::Random::Xorshift128plus rnd(options.seed);

‎crypto/vm/db/DynamicBagOfCellsDb.cpp

+7-1
Original file line numberDiff line numberDiff line change
@@ -107,15 +107,21 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat
107107
td::Result<Ref<Cell>> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override {
108108
return get_cell_info_lazy(level_mask, hash, depth).cell;
109109
}
110-
td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all() const override {
110+
td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all(size_t max_count) const override {
111111
std::vector<std::pair<std::string, std::string>> result;
112112
auto s = loader_->key_value_reader().for_each_in_range("desc", "desd",
113113
[&](const td::Slice &key, const td::Slice &value) {
114+
if (result.size() >= max_count) {
115+
return td::Status::Error("COUNT_LIMIT");
116+
}
114117
if (td::begins_with(key, "desc") && key.size() != 32) {
115118
result.emplace_back(key.str(), value.str());
116119
}
117120
return td::Status::OK();
118121
});
122+
if (s.message() == "COUNT_LIMIT") {
123+
s = td::Status::OK();
124+
}
119125
TRY_STATUS(std::move(s));
120126
return result;
121127
}

‎crypto/vm/db/DynamicBagOfCellsDb.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ class DynamicBagOfCellsDb {
5151
public:
5252
virtual ~DynamicBagOfCellsDb() = default;
5353

54-
virtual td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all() const = 0;
54+
virtual td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all(size_t max_count) const = 0;
5555
virtual td::Result<td::KeyValue::GetStatus> meta_get(td::Slice key, std::string &value) = 0;
5656
virtual td::Status meta_set(td::Slice key, td::Slice value) = 0;
5757
virtual td::Status meta_erase(td::Slice key) = 0;

‎crypto/vm/db/DynamicBagOfCellsDbV2.cpp

+7-1
Original file line numberDiff line numberDiff line change
@@ -765,16 +765,22 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb {
765765
}
766766
}
767767

768-
td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all() const override {
768+
td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all(size_t max_count) const override {
769769
CHECK(meta_db_fixup_.empty());
770770
std::vector<std::pair<std::string, std::string>> result;
771771
auto s = cell_db_reader_->key_value_reader().for_each_in_range(
772772
"desc", "desd", [&](const td::Slice &key, const td::Slice &value) {
773+
if (result.size() >= max_count) {
774+
return td::Status::Error("COUNT_LIMIT");
775+
}
773776
if (td::begins_with(key, "desc") && key.size() != 32) {
774777
result.emplace_back(key.str(), value.str());
775778
}
776779
return td::Status::OK();
777780
});
781+
if (s.message() == "COUNT_LIMIT") {
782+
s = td::Status::OK();
783+
}
778784
TRY_STATUS(std::move(s));
779785
return result;
780786
}

‎crypto/vm/db/InMemoryBagOfCellsDb.cpp

+11-4
Original file line numberDiff line numberDiff line change
@@ -774,8 +774,15 @@ class MetaStorage {
774774
CHECK(p.first.size() != CellTraits::hash_bytes);
775775
}
776776
}
777-
std::vector<std::pair<std::string, std::string>> meta_get_all() const {
778-
return td::transform(meta_, [](const auto &p) { return std::make_pair(p.first, p.second); });
777+
std::vector<std::pair<std::string, std::string>> meta_get_all(size_t max_count) const {
778+
std::vector<std::pair<std::string, std::string>> res;
779+
for (const auto &[k, v] : meta_) {
780+
if (res.size() >= max_count) {
781+
break;
782+
}
783+
res.emplace_back(k, v);
784+
}
785+
return res;
779786
}
780787
KeyValue::GetStatus meta_get(td::Slice key, std::string &value) const {
781788
auto lock = local_access_.lock();
@@ -814,8 +821,8 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb {
814821
: storage_(std::move(storage)), meta_storage_(std::move(meta_storage)) {
815822
}
816823

817-
td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all() const override {
818-
return meta_storage_->meta_get_all();
824+
td::Result<std::vector<std::pair<std::string, std::string>>> meta_get_all(size_t max_count) const override {
825+
return meta_storage_->meta_get_all(max_count);
819826
}
820827
td::Result<KeyValue::GetStatus> meta_get(td::Slice key, std::string &value) override {
821828
CHECK(key.size() != CellTraits::hash_bytes);

‎validator/db/celldb.cpp

+69-53
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,66 @@ struct MergeOperatorAddCellRefcnt : public rocksdb::MergeOperator {
111111
}
112112
};
113113

114+
void CellDbIn::validate_meta() {
115+
LOG(INFO) << "Validating metadata\n";
116+
size_t max_meta_keys_loaded = opts_->get_celldb_in_memory() ? std::numeric_limits<std::size_t>::max() : 10000;
117+
auto meta = boc_->meta_get_all(max_meta_keys_loaded).move_as_ok();
118+
bool partial_check = meta.size() == max_meta_keys_loaded;
119+
if (partial_check) {
120+
LOG(ERROR) << "Too much metadata in the database, do only partial check";
121+
}
122+
size_t missing_roots = 0;
123+
size_t unknown_roots = 0;
124+
std::set<vm::CellHash> root_hashes;
125+
for (auto [k, v] : meta) {
126+
if (k == "desczero") {
127+
continue;
128+
}
129+
auto obj = fetch_tl_object<ton_api::db_celldb_value>(td::BufferSlice{v}, true);
130+
obj.ensure();
131+
auto entry = DbEntry{obj.move_as_ok()};
132+
root_hashes.insert(vm::CellHash::from_slice(entry.root_hash.as_slice()));
133+
auto cell = boc_->load_cell(entry.root_hash.as_slice());
134+
missing_roots += cell.is_error();
135+
LOG_IF(ERROR, cell.is_error()) << "Cannot load root from meta: " << entry.block_id.to_str() << " " << cell.error();
136+
}
137+
138+
// load_known_roots is only supported by InMemory database, so it is ok to check all known roots here
139+
auto known_roots = boc_->load_known_roots().move_as_ok();
140+
for (auto& root : known_roots) {
141+
block::gen::ShardStateUnsplit::Record info;
142+
block::gen::OutMsgQueueInfo::Record qinfo;
143+
block::ShardId shard;
144+
if (!(tlb::unpack_cell(root, info) && shard.deserialize(info.shard_id.write()) &&
145+
tlb::unpack_cell(info.out_msg_queue_info, qinfo))) {
146+
LOG(FATAL) << "cannot create ShardDescr from a root in celldb";
147+
}
148+
if (!partial_check && !root_hashes.contains(root->get_hash())) {
149+
unknown_roots++;
150+
LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no;
151+
constexpr bool delete_unknown_roots = false;
152+
if (delete_unknown_roots) {
153+
vm::CellStorer stor{*cell_db_};
154+
cell_db_->begin_write_batch().ensure();
155+
boc_->dec(root);
156+
boc_->commit(stor).ensure();
157+
cell_db_->commit_write_batch().ensure();
158+
if (!opts_->get_celldb_in_memory()) {
159+
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
160+
}
161+
LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no << " REMOVED";
162+
}
163+
}
164+
}
165+
166+
LOG_IF(ERROR, missing_roots != 0) << "Missing root hashes: " << missing_roots;
167+
LOG_IF(ERROR, unknown_roots != 0) << "Unknown roots: " << unknown_roots;
168+
169+
LOG_IF(FATAL, missing_roots != 0) << "Missing root hashes: " << missing_roots;
170+
LOG_IF(FATAL, unknown_roots != 0) << "Unknown roots: " << unknown_roots;
171+
LOG(INFO) << "Validating metadata: OK\n";
172+
}
173+
114174
void CellDbIn::start_up() {
115175
on_load_callback_ = [actor = std::make_shared<td::actor::ActorOwn<MigrationProxy>>(
116176
td::actor::create_actor<MigrationProxy>("celldbmigration", actor_id(this))),
@@ -141,11 +201,11 @@ void CellDbIn::start_up() {
141201
std::optional<vm::DynamicBagOfCellsDb::CreateV2Options> boc_v2_options;
142202

143203
if (opts_->get_celldb_v2()) {
144-
boc_v2_options =
145-
vm::DynamicBagOfCellsDb::CreateV2Options{.extra_threads = std::clamp(std::thread::hardware_concurrency() / 2, 1u, 8u),
146-
.executor = {},
147-
.cache_ttl_max = 2000,
148-
.cache_size_max = 1000000};
204+
boc_v2_options = vm::DynamicBagOfCellsDb::CreateV2Options{
205+
.extra_threads = std::clamp(std::thread::hardware_concurrency() / 2, 1u, 8u),
206+
.executor = {},
207+
.cache_ttl_max = 2000,
208+
.cache_size_max = 1000000};
149209
size_t min_rocksdb_cache = std::max(size_t{1} << 30, boc_v2_options->cache_size_max * 5000);
150210
if (!o_celldb_cache_size || o_celldb_cache_size.value() < min_rocksdb_cache) {
151211
LOG(WARNING) << "Increase CellDb block cache size to " << td::format::as_size(min_rocksdb_cache) << " from "
@@ -208,54 +268,7 @@ void CellDbIn::start_up() {
208268
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
209269
}
210270

211-
auto meta = boc_->meta_get_all().move_as_ok();
212-
size_t missing_roots = 0;
213-
size_t unknown_roots = 0;
214-
std::set<vm::CellHash> root_hashes;
215-
for (auto [k, v] : meta) {
216-
if (k == "desczero") {
217-
continue;
218-
}
219-
auto obj = fetch_tl_object<ton_api::db_celldb_value>(td::BufferSlice{v}, true);
220-
obj.ensure();
221-
auto entry = DbEntry{obj.move_as_ok()};
222-
root_hashes.insert(vm::CellHash::from_slice(entry.root_hash.as_slice()));
223-
auto cell = boc_->load_cell(entry.root_hash.as_slice());
224-
missing_roots += cell.is_error();
225-
LOG_IF(ERROR, cell.is_error()) << "Cannot load root from meta: " << entry.block_id.to_str() << " " << cell.error();
226-
}
227-
auto known_roots = boc_->load_known_roots().move_as_ok();
228-
for (auto& root : known_roots) {
229-
block::gen::ShardStateUnsplit::Record info;
230-
block::gen::OutMsgQueueInfo::Record qinfo;
231-
block::ShardId shard;
232-
if (!(tlb::unpack_cell(root, info) && shard.deserialize(info.shard_id.write()) &&
233-
tlb::unpack_cell(info.out_msg_queue_info, qinfo))) {
234-
LOG(FATAL) << "cannot create ShardDescr from a root in celldb";
235-
}
236-
if (!root_hashes.contains(root->get_hash())) {
237-
unknown_roots++;
238-
LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no;
239-
constexpr bool delete_unknown_roots = false;
240-
if (delete_unknown_roots) {
241-
vm::CellStorer stor{*cell_db_};
242-
cell_db_->begin_write_batch().ensure();
243-
boc_->dec(root);
244-
boc_->commit(stor).ensure();
245-
cell_db_->commit_write_batch().ensure();
246-
if (!opts_->get_celldb_in_memory()) {
247-
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
248-
}
249-
LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no << " REMOVED";
250-
}
251-
}
252-
}
253-
254-
LOG_IF(ERROR, missing_roots != 1) << "Missing root hashes: " << missing_roots;
255-
LOG_IF(ERROR, unknown_roots != 0) << "Unknown roots: " << unknown_roots;
256-
257-
LOG_IF(FATAL, missing_roots > 1) << "Missing root hashes: " << missing_roots;
258-
LOG_IF(FATAL, unknown_roots != 0) << "Unknown roots: " << unknown_roots;
271+
validate_meta();
259272

260273
alarm_timestamp() = td::Timestamp::in(10.0);
261274

@@ -267,6 +280,9 @@ void CellDbIn::start_up() {
267280
set_block(empty, std::move(e));
268281
boc_->commit(stor);
269282
cell_db_->commit_write_batch().ensure();
283+
if (!opts_->get_celldb_in_memory()) {
284+
boc_->set_loader(std::make_unique<vm::CellLoader>(cell_db_->snapshot(), on_load_callback_)).ensure();
285+
}
270286
}
271287

272288
if (opts_->get_celldb_v2() || opts_->get_celldb_in_memory()) {

‎validator/db/celldb.hpp

+1
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ class CellDbIn : public CellDbBase {
7474
CellDbIn(td::actor::ActorId<RootDb> root_db, td::actor::ActorId<CellDb> parent, std::string path,
7575
td::Ref<ValidatorManagerOptions> opts);
7676

77+
void validate_meta();
7778
void start_up() override;
7879
void alarm() override;
7980

0 commit comments

Comments
 (0)