Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions src/cachemultimap.h
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,16 @@ class CacheMultiMap
return (mapIndex.find(key) != mapIndex.end());
}

const V* Find(const K& key, const V& value) const
{
map_cit mit = mapIndex.find(key);
if (mit == mapIndex.end()) {
return nullptr;
}
it_map_cit it = mit->second.find(value);
return it == mit->second.end() ? nullptr : &it->second->value;
}

bool Get(const K& key, V& value) const
{
map_cit it = mapIndex.find(key);
Expand Down
128 changes: 78 additions & 50 deletions src/governance/governance.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -64,11 +64,72 @@ class ScopedLockBool
};
} // anonymous namespace

namespace governance {
OrphanVoteCache::OrphanVoteCache(size_t max_total, size_t max_per_mn) :
m_max_per_mn{max_per_mn},
m_cache{static_cast<CacheMultiMap<uint256, OrphanVote>::size_type>(max_total)}
{
assert(max_total > 0);
}

bool OrphanVoteCache::Insert(const uint256& parent_hash, const OrphanVote& orphan_vote)
{
if (m_cache.Find(parent_hash, orphan_vote) != nullptr) return true;
Comment thread
PastaPastaPasta marked this conversation as resolved.
Resolved

const COutPoint& outpoint{orphan_vote.vote.GetMasternodeOutpoint()};
if (const auto it{m_count_per_mn.find(outpoint)}; it != m_count_per_mn.end() && it->second >= m_max_per_mn) {
return false;
}
// Evict before m_cache would prune by itself, so the evicted vote's masternode gets its share back.
if (m_cache.GetSize() >= m_cache.GetMaxSize()) {
const auto& oldest{m_cache.GetItemList().back()};
Erase(oldest.key, oldest.value);
}
m_cache.Insert(parent_hash, orphan_vote);
++m_count_per_mn[outpoint];
return true;
}

void OrphanVoteCache::Erase(const uint256& parent_hash, const OrphanVote& orphan_vote)
{
const OrphanVote* stored{m_cache.Find(parent_hash, orphan_vote)};
if (stored == nullptr) return;
// Release the share of the stored entry, which the argument only has to compare equal to. Read it
// first: erasing destroys it.
const COutPoint outpoint{stored->vote.GetMasternodeOutpoint()};
m_cache.Erase(parent_hash, orphan_vote);

const auto it{m_count_per_mn.find(outpoint)};
if (!Assume(it != m_count_per_mn.end())) return;
if (--it->second == 0) m_count_per_mn.erase(it);
}

void OrphanVoteCache::EraseExpired(NodeSeconds now)
{
const auto& items{m_cache.GetItemList()};
for (auto it = items.begin(); it != items.end();) {
const auto& item{*it++};
if (item.value.expiration < now) Erase(item.key, item.value);
}
}

void OrphanVoteCache::GetAll(const uint256& parent_hash, std::vector<OrphanVote>& orphan_votes)
{
m_cache.GetAll(parent_hash, orphan_votes);
}

void OrphanVoteCache::Clear()
{
m_cache.Clear();
m_count_per_mn.clear();
}
} // namespace governance

GovernanceStore::GovernanceStore() :
cs_store(),
mapObjects(),
mapErasedGovernanceObjects(),
cmmapOrphanVotes(MAX_CACHE_SIZE),
m_orphan_votes(governance::MAX_ORPHAN_VOTES, governance::MAX_ORPHAN_VOTES_PER_MN),
mapLastMasternodeObject(),
lastMNListForVotingKeys(std::make_shared<CDeterministicMNList>())
{
Expand Down Expand Up @@ -292,7 +353,7 @@ void CGovernanceManager::CheckOrphanVotes(const std::shared_ptr<CGovernanceObjec

uint256 nHash = Assert(govobj)->GetHash();
std::vector<governance::OrphanVote> orphan_votes;
cmmapOrphanVotes.GetAll(nHash, orphan_votes);
m_orphan_votes.GetAll(nHash, orphan_votes);

ScopedLockBool guard(cs_store, fRateChecksEnabled, false);

Expand All @@ -311,7 +372,7 @@ void CGovernanceManager::CheckOrphanVotes(const std::shared_ptr<CGovernanceObjec
fRemove = true;
}
if (fRemove) {
cmmapOrphanVotes.Erase(nHash, orphan_vote);
m_orphan_votes.Erase(nHash, orphan_vote);
}
}
}
Expand Down Expand Up @@ -401,14 +462,6 @@ void CGovernanceManager::AddGovernanceObjectForTesting(const CGovernanceObject&
mapObjects.emplace(govobj.GetHash(), std::make_shared<CGovernanceObject>(govobj));
}

void CGovernanceManager::AddOrphanVoteForTesting(const CGovernanceVote& vote)
{
AssertLockNotHeld(cs_store);
LOCK(cs_store);
cmmapOrphanVotes.Insert(vote.GetParentHash(),
governance::OrphanVote{vote, Now<NodeSeconds>() + GOVERNANCE_ORPHAN_EXPIRATION_TIME});
}

void CGovernanceManager::CheckAndRemove()
{
AssertLockNotHeld(cs_store);
Expand All @@ -426,6 +479,8 @@ void CGovernanceManager::CheckAndRemove()

ScopedLockBool guard(cs_store, fRateChecksEnabled, false);

m_orphan_votes.EraseExpired(Now<NodeSeconds>());

// Clean up any expired or invalid triggers
m_superblocks.Clean(nCachedBlockHeight);

Expand Down Expand Up @@ -857,8 +912,17 @@ bool CGovernanceManager::ProcessVote(const CGovernanceVote& vote, CGovernanceExc
// No penalty: the vote is signed by a masternode, it just arrived before its parent object,
// which routinely happens during governance sync. Misbehaviour scores never decay.
exception = CGovernanceException(msg, GOVERNANCE_EXCEPTION_WARNING);
if (cmmapOrphanVotes.Insert(nHashGovobj, governance::OrphanVote{vote, Now<NodeSeconds>() + GOVERNANCE_ORPHAN_EXPIRATION_TIME})) {
hashToRequest = nHashGovobj; // Caller should request this object
const auto now{Now<NodeSeconds>()};
if (vote.Time() < now - governance::MAX_ORPHAN_VOTE_AGE) {
LogPrint(BCLog::GOBJECT, "CGovernanceManager::%s -- vote %s is too old to hold as an orphan\n", __func__,
nHashVote.ToString());
} else if (m_orphan_votes.Insert(nHashGovobj,
governance::OrphanVote{vote, now + GOVERNANCE_ORPHAN_EXPIRATION_TIME})) {
// Even if we already held this vote: every peer that relays it can serve the parent.
hashToRequest = nHashGovobj;
} else {
LogPrint(BCLog::GOBJECT, "CGovernanceManager::%s -- MN outpoint %s holds its share of orphan votes, dropping vote %s\n",
__func__, vote.GetMasternodeOutpoint().ToStringShort(), nHashVote.ToString());
}
LogPrint(BCLog::GOBJECT, "%s\n", msg);
return false;
Expand Down Expand Up @@ -1034,7 +1098,7 @@ void GovernanceStore::Clear()
LOCK(cs_store);
mapObjects.clear();
mapErasedGovernanceObjects.clear();
cmmapOrphanVotes.Clear();
m_orphan_votes.Clear();
mapLastMasternodeObject.clear();
lastMNListForVotingKeys = std::make_shared<CDeterministicMNList>();
}
Expand Down Expand Up @@ -1144,41 +1208,6 @@ void CGovernanceManager::UpdatedBlockTip(const CBlockIndex* pindex)
m_superblocks.ExecuteBestSuperblock(m_dmnman.GetListAtChainTip(), pindex->nHeight);
}

std::vector<uint256> CGovernanceManager::GetOrphanVoteObjectHashes()
{
LOCK(cs_store);

const auto now{Now<NodeSeconds>()};

// Clean up expired orphan votes
const vote_cmm_t::list_t& items = cmmapOrphanVotes.GetItemList();
for (auto it = items.begin(); it != items.end();) {
auto prevIt = it;
++it;
if (prevIt->value.expiration < now) {
cmmapOrphanVotes.Erase(prevIt->key, prevIt->value);
}
}

// Get hashes of objects we don't have yet
std::vector<uint256> vecHashesFiltered;
std::vector<uint256> vecHashes;
cmmapOrphanVotes.GetKeys(vecHashes);
for (const uint256& nHash : vecHashes) {
if (mapObjects.find(nHash) == mapObjects.end()) {
vecHashesFiltered.push_back(nHash);
}
}

// A random subset, so that parent hashes ground to sort first cannot pin which ones are requested
if (vecHashesFiltered.size() > MAX_ORPHAN_OBJECT_REQUESTS) {
Shuffle(vecHashesFiltered.begin(), vecHashesFiltered.end(), FastRandomContext());
vecHashesFiltered.resize(MAX_ORPHAN_OBJECT_REQUESTS);
}

return vecHashesFiltered;
}

void CGovernanceManager::RemoveInvalidVotes()
{
AssertLockHeld(cs_store);
Expand Down Expand Up @@ -1222,7 +1251,6 @@ void CGovernanceManager::RemoveInvalidVotes()
}
for (const auto& voteHash : removed) {
cmapVoteToObject.Erase(voteHash);
cmmapOrphanVotes.Erase(voteHash);
}
}
}
Expand Down
90 changes: 76 additions & 14 deletions src/governance/governance.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,19 @@
#include <governance/vote.h>
#include <kernel/cs_main.h> // IWYU pragma: export
#include <primitives/transaction.h>
#include <serialize.h>
#include <sync.h>
#include <util/fs.h>
#include <util/time.h>

#include <atomic>
#include <chrono>
#include <limits>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <utility>
#include <vector>

class CBloomFilter;
Expand Down Expand Up @@ -59,8 +62,76 @@ struct OrphanVote {

inline bool operator<(const OrphanVote& lhs, const OrphanVote& rhs)
{
// Equal signed fields may arrive authenticated by different voting/operator keys.
if (lhs.vote == rhs.vote) return lhs.vote.GetSignatureSize() < rhs.vote.GetSignatureSize();
return lhs.vote < rhs.vote;
}

// Orphan votes are not scarce: one masternode key can sign any number of votes naming parents that
// do not exist. Each masternode is therefore limited to its own share of the cache, and the cache as
// a whole holds about 40 MB at most (roughly 800 bytes per entry). Filling it takes the keys of 500
// masternodes, because only recently signed votes are held.
static constexpr size_t MAX_ORPHAN_VOTES{50'000};
static constexpr size_t MAX_ORPHAN_VOTES_PER_MN{100};
// A vote stays valid for as long as its masternode's key does, so anyone can replay a masternode's
// old votes for objects we no longer have. Only recent votes are held as orphans, or such replays
// would use up the masternode's share of the orphan cache.
static constexpr std::chrono::seconds MAX_ORPHAN_VOTE_AGE{std::chrono::hours{1}};

/** Votes waiting for their parent object, keyed by parent hash, with the oldest evicted once the
* cache is full and each masternode limited to a share of it. */
class OrphanVoteCache
{
public:
explicit OrphanVoteCache(size_t max_total, size_t max_per_mn);

/** Returns whether the vote is held, either newly or because it already was. A new vote is
* dropped if its masternode already holds max_per_mn orphan votes. */
[[nodiscard]] bool Insert(const uint256& parent_hash, const OrphanVote& orphan_vote);
void Erase(const uint256& parent_hash, const OrphanVote& orphan_vote);
void EraseExpired(NodeSeconds now);
void GetAll(const uint256& parent_hash, std::vector<OrphanVote>& orphan_votes);
void Clear();

template <typename Stream>
void Serialize(Stream& s) const
{
s << m_cache;
}

template <typename Stream>
void Unserialize(Stream& s)
{
// Read entry by entry rather than adopt the stored cache: a file from an earlier version can
// hold a million entries. They are stored newest first, so keep the newest that fit the
// bounds, skipping those ProcessVote() would not hold, then insert them oldest first.
using Stored = CacheMultiMap<uint256, OrphanVote>;
Stored::size_type stored_capacity;
s >> stored_capacity;
Clear();
const auto now{Now<NodeSeconds>()};
std::vector<Stored::item_t> kept;
std::map<COutPoint, size_t> kept_per_mn;
for (uint64_t remaining{ReadCompactSize(s)}; remaining > 0; --remaining) {
Stored::item_t item;
s >> item;
if (item.value.expiration < now || item.value.vote.Time() < now - MAX_ORPHAN_VOTE_AGE) continue;
if (kept.size() >= m_cache.GetMaxSize()) continue;
if (size_t& count{kept_per_mn[item.value.vote.GetMasternodeOutpoint()]}; count < m_max_per_mn) {
++count;
kept.push_back(std::move(item));
}
}
for (auto it = kept.rbegin(); it != kept.rend(); ++it) {
(void)Insert(it->key, it->value);
Comment thread
PastaPastaPasta marked this conversation as resolved.
Resolved
}
}

private:
const size_t m_max_per_mn;
CacheMultiMap<uint256, OrphanVote> m_cache;
std::map<COutPoint, size_t> m_count_per_mn;
};
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
} // namespace governance

static constexpr int RATE_BUFFER_SIZE = 5;
Expand Down Expand Up @@ -176,7 +247,6 @@ class GovernanceStore
};

using txout_m_t = std::map<COutPoint, last_object_rec>;
using vote_cmm_t = CacheMultiMap<uint256, governance::OrphanVote>;

protected:
static constexpr int MAX_CACHE_SIZE = 1000000;
Expand All @@ -192,7 +262,7 @@ class GovernanceStore
// key - governance object's hash
// value - expiration time for deleted objects
std::map<uint256, int64_t> mapErasedGovernanceObjects GUARDED_BY(cs_store);
vote_cmm_t cmmapOrphanVotes GUARDED_BY(cs_store);
governance::OrphanVoteCache m_orphan_votes GUARDED_BY(cs_store);
txout_m_t mapLastMasternodeObject GUARDED_BY(cs_store);
// used to check for changed voting keys
std::shared_ptr<CDeterministicMNList> lastMNListForVotingKeys GUARDED_BY(cs_store);
Expand All @@ -207,7 +277,7 @@ class GovernanceStore
LOCK(cs_store);
s << SERIALIZATION_VERSION_STRING
<< mapErasedGovernanceObjects
<< cmmapOrphanVotes
<< m_orphan_votes
<< mapObjects
<< mapLastMasternodeObject
<< *lastMNListForVotingKeys;
Expand All @@ -226,7 +296,7 @@ class GovernanceStore
}

s >> mapErasedGovernanceObjects
>> cmmapOrphanVotes
>> m_orphan_votes
>> mapObjects
>> mapLastMasternodeObject
>> *lastMNListForVotingKeys;
Expand Down Expand Up @@ -337,8 +407,6 @@ class CGovernanceManager : public GovernanceStore
* running collateral or chain validation. */
void AddGovernanceObjectForTesting(const CGovernanceObject& govobj)
EXCLUSIVE_LOCKS_REQUIRED(!cs_store);
/** Test-only helper: caches a vote as an orphan of its parent without validating it. */
void AddOrphanVoteForTesting(const CGovernanceVote& vote) EXCLUSIVE_LOCKS_REQUIRED(!cs_store);

// Thread-safe accessors
bool HaveObjectForHash(const uint256& nHash) const
Expand All @@ -365,13 +433,6 @@ class CGovernanceManager : public GovernanceStore
// Used by NetGovernance
std::vector<CInv> FetchRelayInventory() EXCLUSIVE_LOCKS_REQUIRED(!cs_relay);
void CheckAndRemove() EXCLUSIVE_LOCKS_REQUIRED(!cs_store);
/** Upper bound on the orphan parent hashes GetOrphanVoteObjectHashes() returns per call. Each one is
* re-requested from every peer, and one masternode key can sign votes for any number of invented
* parent hashes. */
static constexpr size_t MAX_ORPHAN_OBJECT_REQUESTS{100};
/** Get hashes of governance objects for which we have orphan votes, at most
* MAX_ORPHAN_OBJECT_REQUESTS of them chosen at random. Also cleans up expired orphans. */
[[nodiscard]] std::vector<uint256> GetOrphanVoteObjectHashes() EXCLUSIVE_LOCKS_REQUIRED(!cs_store);
std::pair<std::vector<uint256>, std::vector<uint256>> FetchGovernanceObjectVotes(
size_t peers_per_hash_max, int64_t now, std::map<uint256, std::map<CService, int64_t>>& map_asked_recently) const
EXCLUSIVE_LOCKS_REQUIRED(!cs_store);
Expand All @@ -387,7 +448,8 @@ class CGovernanceManager : public GovernanceStore

CDeterministicMNManager& GetMNManager();
/** Process a governance vote. Returns true on success.
* If the vote is for an unknown object (orphan), hashToRequest is set to the object hash. */
* If the vote is for an unknown object (orphan) and is held until that object arrives,
* hashToRequest is set to the object hash. */
bool ProcessVote(const CGovernanceVote& vote, CGovernanceException& exception, uint256& hashToRequest)
EXCLUSIVE_LOCKS_REQUIRED(!cs_store);

Expand Down
Loading
Loading