Skip to content

Commit

Permalink
Adding a memory container for blocks once the initial bootstrap thres…
Browse files Browse the repository at this point in the history 
…hold is reached. Blocks are pruned in fifo order.
  • Loading branch information
@clemahieu
clemahieu committed Jun 8, 2022
1 parent 9b5bae6 commit 0061009
Show file tree
Hide file tree
Showing 5 changed files with 142 additions and 21 deletions.
1 change: 1 addition & 0 deletions nano/node/node.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ nano::node::node (boost::asio::io_context & io_ctx_a, boost::filesystem::path co
startup_time (std::chrono::steady_clock::now ()),
node_seq (seq)
{
store.unchecked.use_memory = [this] () { return ledger.bootstrap_weight_reached (); };
if (!init_error ())
{
telemetry->start ();
Expand Down
5 changes: 5 additions & 0 deletions nano/secure/common.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -894,6 +894,11 @@ bool nano::unchecked_key::operator== (nano::unchecked_key const & other_a) const
return previous == other_a.previous && hash == other_a.hash;
}

bool nano::unchecked_key::operator< (nano::unchecked_key const & other_a) const
{
return previous != other_a.previous ? previous < other_a.previous : hash < other_a.hash;
}

nano::block_hash const & nano::unchecked_key::key () const
{
return previous;
Expand Down
1 change: 1 addition & 0 deletions nano/secure/common.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@ class unchecked_key final
unchecked_key (nano::uint512_union const &);
bool deserialize (nano::stream &);
bool operator== (nano::unchecked_key const &) const;
bool operator< (nano::unchecked_key const &) const;
nano::block_hash const & key () const;
nano::block_hash previous{ 0 };
nano::block_hash hash{ 0 };
Expand Down
2 changes: 2 additions & 0 deletions nano/secure/store.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -757,6 +757,8 @@ class unchecked_store
nano::transaction const & tx, nano::hash_or_account const & dependency, std::function<void (nano::unchecked_key const &, nano::unchecked_info const &)> action, std::function<bool ()> predicate = [] () { return true; })
= 0;
virtual size_t count (nano::transaction const &) = 0;

std::function<bool ()> use_memory = [] () { return true; };
};

/**
Expand Down
154 changes: 133 additions & 21 deletions nano/secure/store/unchecked_store_partial.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,12 @@

#include <nano/secure/store_partial.hpp>

#include <boost/multi_index/member.hpp>
#include <boost/multi_index/ordered_index.hpp>
#include <boost/multi_index/random_access_index.hpp>

namespace mi = boost::multi_index;

namespace
{
template <typename T>
Expand All @@ -19,66 +25,149 @@ void release_assert_success (store_partial<Val, Derived_Store> const &, int cons
template <typename Val, typename Derived_Store>
class unchecked_store_partial : public unchecked_store
{
class tag_random_access
{
};
class tag_root
{
};

private:
nano::store_partial<Val, Derived_Store> & store;

friend void release_assert_success<Val, Derived_Store> (store_partial<Val, Derived_Store> const &, int const);
static size_t constexpr mem_block_count_max = 256'000;

public:
unchecked_store_partial (nano::store_partial<Val, Derived_Store> & store_a) :
store (store_a){};

void clear (nano::write_transaction const & transaction_a) override
{
auto status = store.drop (transaction_a, tables::unchecked);
release_assert_success (store, status);
nano::lock_guard<std::recursive_mutex> lock{ mutex };
if (entries == nullptr)
{
auto status = store.drop (transaction_a, tables::unchecked);
release_assert_success (store, status);
}
else
{
entries->clear ();
}
}

void put (nano::write_transaction const & transaction_a, nano::unchecked_key const & key_a, nano::unchecked_info const & info_a) override
{
if (get (transaction_a, key_a.previous).size () > 1)
return;
nano::db_val<Val> info (info_a);
auto status (store.put (transaction_a, tables::unchecked, key_a, info));
release_assert_success (store, status);
nano::lock_guard<std::recursive_mutex> lock{ mutex };
if (entries == nullptr && use_memory ())
{
auto entries_new = std::make_unique<typename decltype (entries)::element_type> ();
for_each (
transaction_a, [&entries_new] (nano::unchecked_key const & key, nano::unchecked_info const & info) { entries_new->template get<tag_root> ().insert ({ key, info }); }, [&] () { return entries_new->size () < mem_block_count_max; });
clear (transaction_a);
entries = std::move (entries_new);
}
if (entries == nullptr)
{
nano::db_val<Val> info (info_a);
auto status (store.put (transaction_a, tables::unchecked, key_a, info));
release_assert_success (store, status);
}
else
{
entries->template get<tag_root> ().insert ({ key_a, info_a });
while (entries->size () > mem_block_count_max)
{
entries->template get<tag_random_access> ().pop_front ();
}
}
}

void put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a, std::shared_ptr<nano::block> const & block_a) override
{
nano::unchecked_key key (hash_a, block_a->hash ());
nano::unchecked_info info (block_a, block_a->account (), nano::seconds_since_epoch (), nano::signature_verification::unknown);
put (transaction_a, key, info);
nano::lock_guard<std::recursive_mutex> lock{ mutex };
if (entries == nullptr)
{
nano::unchecked_key key (hash_a, block_a->hash ());
nano::unchecked_info info{ block_a, block_a->account (), nano::seconds_since_epoch (), nano::signature_verification::unknown };
put (transaction_a, key, info);
}
else
{
nano::unchecked_key key (hash_a, block_a->hash ());
nano::unchecked_info info{ block_a, block_a->account (), nano::seconds_since_epoch (), nano::signature_verification::unknown };
put (transaction_a, key, info);
}
}

bool exists (nano::transaction const & transaction_a, nano::unchecked_key const & unchecked_key_a) override
{
nano::db_val<Val> value;
auto status (store.get (transaction_a, tables::unchecked, nano::db_val<Val> (unchecked_key_a), value));
release_assert (store.success (status) || store.not_found (status));
return (store.success (status));
nano::lock_guard<std::recursive_mutex> lock{ mutex };
if (entries == nullptr)
{
nano::db_val<Val> value;
auto status (store.get (transaction_a, tables::unchecked, nano::db_val<Val> (unchecked_key_a), value));
release_assert (store.success (status) || store.not_found (status));
return (store.success (status));
}
else
{
return entries->template get<tag_root> ().count (unchecked_key_a) != 0;
}
}

void del (nano::write_transaction const & transaction_a, nano::unchecked_key const & key_a) override
{
auto status (store.del (transaction_a, tables::unchecked, key_a));
release_assert_success (store, status);
nano::lock_guard<std::recursive_mutex> lock{ mutex };
if (entries == nullptr)
{
auto status (store.del (transaction_a, tables::unchecked, key_a));
release_assert_success (store, status);
}
else
{
auto erased = entries->template get<tag_root> ().erase (key_a);
release_assert (erased);
}
}

void for_each (
nano::transaction const & tx, std::function<void (nano::unchecked_key const &, nano::unchecked_info const &)> action, std::function<bool ()> predicate = [] () { return true; }) override
{
for (auto i = store.template make_iterator<nano::unchecked_key, nano::unchecked_info> (tx, tables::unchecked), n = nano::store_iterator<nano::unchecked_key, nano::unchecked_info> (nullptr); predicate () && i != n; ++i)
nano::lock_guard<std::recursive_mutex> lock{ mutex };
if (entries == nullptr)
{
for (auto i = store.template make_iterator<nano::unchecked_key, nano::unchecked_info> (tx, tables::unchecked), n = nano::store_iterator<nano::unchecked_key, nano::unchecked_info> (nullptr); predicate () && i != n; ++i)
{
action (i->first, i->second);
}
}
else
{
action (i->first, i->second);
for (auto i = entries->begin (), n = entries->end (); predicate () && i != n; ++i)
{
action (i->key, i->info);
}
}
}

void for_each (
nano::transaction const & tx, nano::hash_or_account const & dependency, std::function<void (nano::unchecked_key const &, nano::unchecked_info const &)> action, std::function<bool ()> predicate = [] () { return true; }) override
{
for (auto i = store.template make_iterator<nano::unchecked_key, nano::unchecked_info> (tx, tables::unchecked, nano::unchecked_key{ dependency, 0 }), n = nano::store_iterator<nano::unchecked_key, nano::unchecked_info> (nullptr); predicate () && i->first.key () == dependency && i != n; ++i)
nano::lock_guard<std::recursive_mutex> lock{ mutex };
if (entries == nullptr)
{
action (i->first, i->second);
for (auto i = store.template make_iterator<nano::unchecked_key, nano::unchecked_info> (tx, tables::unchecked, nano::unchecked_key{ dependency, 0 }), n = nano::store_iterator<nano::unchecked_key, nano::unchecked_info> (nullptr); predicate () && i != n && i->first.key () == dependency; ++i)
{
action (i->first, i->second);
}
}
else
{
for (auto i = entries->template get<tag_root> ().lower_bound (nano::unchecked_key{ dependency, 0 }), n = entries->template get<tag_root> ().end (); predicate () && i != n && i->key.key () == dependency; ++i)
{
action (i->key, i->info);
}
}
}

Expand All @@ -93,8 +182,31 @@ class unchecked_store_partial : public unchecked_store

size_t count (nano::transaction const & transaction_a) override
{
return store.count (transaction_a, tables::unchecked);
nano::lock_guard<std::recursive_mutex> lock{ mutex };
if (entries == nullptr)
{
return store.count (transaction_a, tables::unchecked);
}
else
{
return entries->size ();
}
}

private:
class entry
{
public:
nano::unchecked_key key;
nano::unchecked_info info;
};
std::recursive_mutex mutex;
std::unique_ptr<mi::multi_index_container<entry,
mi::indexed_by<
mi::random_access<mi::tag<tag_random_access>>,
mi::ordered_unique<mi::tag<tag_root>,
mi::member<entry, nano::unchecked_key, &entry::key>>>>>
entries;
};

}

0 comments on commit 0061009

Please # to comment.