Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 12 additions & 13 deletions src/addrdb.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -52,16 +52,16 @@ bool SerializeDB(Stream& stream, const Data& data)
}

template <typename Data>
bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data& data, int version)
bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data& data)
{
// Generate random temporary filename
const uint16_t randv{GetRand<uint16_t>()};
std::string tmpfn = strprintf("%s.%04x", prefix, randv);

// open temp output file, and associate with CAutoFile
// open temp output file
fs::path pathTmp = gArgs.GetDataDirNet() / fs::u8path(tmpfn);
FILE *file = fsbridge::fopen(pathTmp, "wb");
CAutoFile fileout(file, SER_DISK, version);
AutoFile fileout{file};
if (fileout.IsNull()) {
fileout.fclose();
remove(pathTmp);
Expand Down Expand Up @@ -94,9 +94,9 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data
}

template <typename Stream, typename Data>
void DeserializeDB(Stream& stream, Data& data, bool fCheckSum = true)
void DeserializeDB(Stream& stream, Data&& data, bool fCheckSum = true)
{
CHashVerifier<Stream> verifier(&stream);
HashVerifier verifier{stream};
// de-serialize file header (network specific magic number) and ..
unsigned char pchMsgTmp[4];
verifier >> pchMsgTmp;
Expand All @@ -119,11 +119,10 @@ void DeserializeDB(Stream& stream, Data& data, bool fCheckSum = true)
}

template <typename Data>
void DeserializeFileDB(const fs::path& path, Data& data, int version)
void DeserializeFileDB(const fs::path& path, Data&& data)
{
// open input file, and associate with CAutoFile
FILE* file = fsbridge::fopen(path, "rb");
CAutoFile filein(file, SER_DISK, version);
AutoFile filein{file};
if (filein.IsNull()) {
throw DbNotFoundError{};
}
Expand Down Expand Up @@ -183,10 +182,10 @@ bool CBanDB::Read(banmap_t& banSet)
bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr)
{
const auto pathAddr = gArgs.GetDataDirNet() / "peers.dat";
return SerializeFileDB("peers", pathAddr, addr, CLIENT_VERSION);
return SerializeFileDB("peers", pathAddr, addr);
}

void ReadFromStream(AddrMan& addr, CDataStream& ssPeers)
void ReadFromStream(AddrMan& addr, DataStream& ssPeers)
{
DeserializeDB(ssPeers, addr, false);
}
Expand All @@ -199,7 +198,7 @@ util::Result<std::unique_ptr<AddrMan>> LoadAddrman(const NetGroupManager& netgro
const auto start{SteadyClock::now()};
const auto path_addr{gArgs.GetDataDirNet() / "peers.dat"};
try {
DeserializeFileDB(path_addr, *addrman, CLIENT_VERSION);
DeserializeFileDB(path_addr, *addrman);
LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman->Size(), Ticks<std::chrono::milliseconds>(SteadyClock::now() - start));
} catch (const DbNotFoundError&) {
// Addrman can be in an inconsistent state after failure, reset it
Expand Down Expand Up @@ -233,14 +232,14 @@ util::Result<std::unique_ptr<AddrMan>> LoadAddrman(const NetGroupManager& netgro
void DumpAnchors(const fs::path& anchors_db_path, const std::vector<CAddress>& anchors)
{
LOG_TIME_SECONDS(strprintf("Flush %d outbound block-relay-only peer addresses to anchors.dat", anchors.size()));
SerializeFileDB("anchors", anchors_db_path, anchors, CLIENT_VERSION | ADDRV2_FORMAT);
SerializeFileDB("anchors", anchors_db_path, CAddress::V2_DISK(anchors));
}

std::vector<CAddress> ReadAnchors(const fs::path& anchors_db_path)
{
std::vector<CAddress> anchors;
try {
DeserializeFileDB(anchors_db_path, anchors, CLIENT_VERSION | ADDRV2_FORMAT);
DeserializeFileDB(anchors_db_path, CAddress::V2_DISK(anchors));
LogPrintf("Loaded %i addresses from %s\n", anchors.size(), fs::quoted(fs::PathToString(anchors_db_path.filename())));
} catch (const std::exception&) {
anchors.clear();
Expand Down
7 changes: 4 additions & 3 deletions src/addrdb.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,14 @@
class ArgsManager;
class AddrMan;
class CAddress;
class CDataStream;
class DataStream;
class NetGroupManager;
struct bilingual_str;

bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr);
/** Only used by tests. */
void ReadFromStream(AddrMan& addr, CDataStream& ssPeers);
void ReadFromStream(AddrMan& addr, DataStream& ssPeers);

bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr);

/** Access to the banlist database (banlist.json) */
class CBanDB
Expand Down
25 changes: 9 additions & 16 deletions src/addrman.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -175,8 +175,7 @@ void AddrManImpl::Serialize(Stream& s_) const
*/

// Always serialize in the latest version (FILE_FORMAT).

OverrideStream<Stream> s(&s_, s_.GetType(), s_.GetVersion() | ADDRV2_FORMAT);
ParamsStream s{CAddress::V2_DISK, s_};

s << static_cast<uint8_t>(FILE_FORMAT);

Expand Down Expand Up @@ -240,14 +239,8 @@ void AddrManImpl::Unserialize(Stream& s_)
Format format;
s_ >> Using<CustomUintFormatter<1>>(format);

int stream_version = s_.GetVersion();
if (format >= Format::V3_BIP155) {
// Add ADDRV2_FORMAT to the version so that the CNetAddr and CAddress
// unserialize methods know that an address in addrv2 format is coming.
stream_version |= ADDRV2_FORMAT;
}

OverrideStream<Stream> s(&s_, s_.GetType(), stream_version);
const auto ser_params = (format >= Format::V3_BIP155 ? CAddress::V2_DISK : CAddress::V1_DISK);
ParamsStream s{ser_params, s_};

uint8_t compat;
s >> compat;
Expand Down Expand Up @@ -1310,12 +1303,12 @@ void AddrMan::Unserialize(Stream& s_)
}

// explicit instantiation
template void AddrMan::Serialize(HashedSourceWriter<CAutoFile>& s) const;
template void AddrMan::Serialize(CDataStream& s) const;
template void AddrMan::Unserialize(CAutoFile& s);
template void AddrMan::Unserialize(CHashVerifier<CAutoFile>& s);
template void AddrMan::Unserialize(CDataStream& s);
template void AddrMan::Unserialize(CHashVerifier<CDataStream>& s);
template void AddrMan::Serialize(HashedSourceWriter<AutoFile>&) const;
template void AddrMan::Serialize(DataStream&) const;
template void AddrMan::Unserialize(AutoFile&);
template void AddrMan::Unserialize(HashVerifier<AutoFile>&);
template void AddrMan::Unserialize(DataStream&);
template void AddrMan::Unserialize(HashVerifier<DataStream>&);

size_t AddrMan::Size(std::optional<Network> net, std::optional<bool> in_new) const
{
Expand Down
3 changes: 1 addition & 2 deletions src/addrman_impl.h
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,7 @@ class AddrInfo : public CAddress

SERIALIZE_METHODS(AddrInfo, obj)
{
READWRITEAS(CAddress, obj);
READWRITE(obj.source, Using<ChronoFormatter<int64_t>>(obj.m_last_success), obj.nAttempts);
READWRITE(AsBase<CAddress>(obj), obj.source, Using<ChronoFormatter<int64_t>>(obj.m_last_success), obj.nAttempts);
}

AddrInfo(const CAddress &addrIn, const CNetAddr &addrSource) : CAddress(addrIn), source(addrSource)
Expand Down
17 changes: 8 additions & 9 deletions src/evo/deterministicmns.h
Original file line number Diff line number Diff line change
Expand Up @@ -118,13 +118,13 @@ void UnserializeImmerMap(Stream& is, immer::map<K, T, Hash, Equal>& m)
// For some reason the compiler is not able to choose the correct Serialize/Deserialize methods without a specialized
// version of SerReadWrite. It otherwise always chooses the version that calls a.Serialize()
template<typename Stream, typename K, typename T, typename Hash, typename Equal>
inline void SerReadWrite(Stream& s, const immer::map<K, T, Hash, Equal>& m, CSerActionSerialize ser_action)
inline void SerReadWrite(Stream& s, const immer::map<K, T, Hash, Equal>& m, ActionSerialize ser_action)
{
::SerializeImmerMap(s, m);
}

template<typename Stream, typename K, typename T, typename Hash, typename Equal>
inline void SerReadWrite(Stream& s, immer::map<K, T, Hash, Equal>& obj, CSerActionUnserialize ser_action)
inline void SerReadWrite(Stream& s, immer::map<K, T, Hash, Equal>& obj, ActionUnserialize ser_action)
{
::UnserializeImmerMap(s, obj);
}
Expand Down Expand Up @@ -234,7 +234,7 @@ class CDeterministicMNList
template<typename Stream>
void Serialize(Stream& s) const
{
const_cast<CDeterministicMNList*>(this)->SerializationOpBase(s, CSerActionSerialize());
const_cast<CDeterministicMNList*>(this)->SerializationOpBase(s, ActionSerialize());

// Serialize the map as a vector
WriteCompactSize(s, mnMap.size());
Expand All @@ -248,7 +248,7 @@ class CDeterministicMNList
{
Clear();

SerializationOpBase(s, CSerActionUnserialize());
SerializationOpBase(s, ActionUnserialize());

for (size_t to_read = ReadCompactSize(s); to_read > 0; --to_read) {
AddMN(std::make_shared<CDeterministicMN>(deserialize, s), /*fBumpTotalCount=*/false);
Expand Down Expand Up @@ -534,17 +534,16 @@ class CDeterministicMNList
DMNL_NO_TEMPLATE(NetInfoInterface);
DMNL_NO_TEMPLATE(std::shared_ptr<NetInfoInterface>);
#undef DMNL_NO_TEMPLATE
int ser_version{PROTOCOL_VERSION};
if constexpr (std::is_same_v<std::decay_t<T>, CService>) {
// Special handling is required if we're using addresses that can only be (de)serialized using
// ADDRv2. Without this step, the address gets truncated, the hashmap gets contaminated with
// an invalid entry and subsequent attempts at registering ADDRv2 entries get blocked. We cannot
// apply this treatment ADDRv1 compatible addresses for backwards compatibility with the existing map.
if (!v.IsAddrV1Compatible()) {
ser_version |= ADDRV2_FORMAT;
}
const auto& ser_params{v.IsAddrV1Compatible() ? CNetAddr::V1 : CNetAddr::V2};
return ::SerializeHash(WithParams(ser_params, v), /*nType=*/SER_GETHASH, /*nVersion=*/PROTOCOL_VERSION);
} else {
return ::SerializeHash(v, /*nType=*/SER_GETHASH, /*nVersion=*/PROTOCOL_VERSION);
}
return ::SerializeHash(v, /*nType=*/SER_GETHASH, /*nVersion=*/ser_version);
}
template <typename T>
[[nodiscard]] bool AddUniqueProperty(const CDeterministicMN& dmn, const T& v)
Expand Down
18 changes: 8 additions & 10 deletions src/evo/netinfo.h
Original file line number Diff line number Diff line change
Expand Up @@ -225,12 +225,11 @@ class NetInfoEntry
bool operator!=(const NetInfoEntry& rhs) const { return !(*this == rhs); }

template <typename Stream>
void Serialize(Stream& s_) const
void Serialize(Stream& s) const
{
OverrideStream<Stream> s(&s_, /*nType=*/0, s_.GetVersion() | ADDRV2_FORMAT);
if (const auto* data_ptr_service{std::get_if<CService>(&m_data)};
m_type == NetInfoType::Service && data_ptr_service && data_ptr_service->IsValid()) {
s << m_type << *data_ptr_service;
s << m_type << CNetAddr::V2(*data_ptr_service);
} else if (const auto* data_ptr_domain{std::get_if<DomainPort>(&m_data)};
m_type == NetInfoType::Domain && data_ptr_domain && data_ptr_domain->IsValid()) {
s << m_type << *data_ptr_domain;
Expand All @@ -240,14 +239,13 @@ class NetInfoEntry
}

template <typename Stream>
void Unserialize(Stream& s_)
void Unserialize(Stream& s)
{
OverrideStream<Stream> s(&s_, /*nType=*/0, s_.GetVersion() | ADDRV2_FORMAT);
s >> m_type;
if (m_type == NetInfoType::Service) {
try {
auto& service{m_data.emplace<CService>()};
s >> service;
s >> CNetAddr::V2(service);
if (!service.IsValid()) { Clear(); } // Invalid CService, mark as invalid
} catch (const std::ios_base::failure&) { Clear(); } // Deser failed, mark as invalid
} else if (m_type == NetInfoType::Domain) {
Expand Down Expand Up @@ -327,23 +325,23 @@ class MnNetInfo final : public NetInfoInterface
void Serialize(Stream& s) const
{
if (const auto service_opt{m_addr.GetAddrPort()}) {
s << *service_opt;
s << CNetAddr::V1(*service_opt);
} else {
s << CService{};
s << CNetAddr::V1(CService{});
}
}

// cppcheck-suppress functionStatic
void Serialize(CSizeComputer& s) const
{
s.seek(::GetSerializeSize(CService{}, s.GetVersion()));
s.seek(::GetSerializeSize(CNetAddr::V1(CService{}), s.GetVersion()));
}

template <typename Stream>
void Unserialize(Stream& s)
{
CService service;
s >> service;
s >> CNetAddr::V1(service);
m_addr = NetInfoEntry{service};
}

Expand Down
4 changes: 2 additions & 2 deletions src/evo/snapshot.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,8 @@ void ValidateHashPrefixRuns(std::vector<uint64_t>& prefixes)
template <typename T>
void ValidateMNEncoding(const T& obj)
{
// Use the decoder's actual accounting, including version-dependent fields
// and OverrideStream wrappers, rather than maintaining a second size formula.
// Use the decoder's actual accounting, including version-dependent fields,
// rather than maintaining a second size formula.
CDataStream encoded{SER_DISK, CLIENT_VERSION};
encoded << obj;
SnapshotBoundedInput bounded{encoded, EVO_SNAPSHOT_MAX_MN_COMPACT_ITEMS};
Expand Down
22 changes: 6 additions & 16 deletions src/evo/snapshot.h
Original file line number Diff line number Diff line change
Expand Up @@ -167,15 +167,16 @@ template <typename Stream>
class SnapshotBoundedInput
{
private:
CHashVerifier<Stream> m_stream;
Stream& m_source;
HashVerifier<Stream> m_stream;
uint64_t m_compact_budget;

public:
SnapshotBoundedInput(Stream& stream, uint64_t compact_budget) :
m_stream{&stream}, m_compact_budget{compact_budget} {}
m_source{stream}, m_stream{stream}, m_compact_budget{compact_budget} {}

int GetType() const { return m_stream.GetType(); }
int GetVersion() const { return m_stream.GetVersion(); }
int GetType() const { return m_source.GetType(); }
int GetVersion() const { return m_source.GetVersion(); }
void read(Span<std::byte> dst) { m_stream.read(dst); }
void ignore(size_t size) { m_stream.ignore(size); }

Expand Down Expand Up @@ -214,17 +215,6 @@ uint64_t ReadCompactSize(SnapshotBoundedInput<Stream>& stream)
return stream.ReadBudgetedCompactSize();
}

/**
* NetInfoEntry overrides the stream version while decoding its payload. Keep
* the snapshot-local CompactSize budget visible through that transparent
* wrapper so strings are rejected before their deserializer resizes them.
*/
template <typename Stream>
uint64_t ReadCompactSize(OverrideStream<SnapshotBoundedInput<Stream>>& stream)
{
return stream.GetStream().ReadBudgetedCompactSize();
}

/**
* Snapshot-local canonical deterministic-MN encoding.
*
Expand Down Expand Up @@ -431,7 +421,7 @@ MinedQuorumCommitment ReadMinedQuorumCommitment(Stream& s)
ReadFixedBitSet(s, commitment.validMembers, valid_members_size);
const bool legacy{commitment.nVersion == llmq::CFinalCommitment::LEGACY_BLS_NON_INDEXED_QUORUM_VERSION ||
commitment.nVersion == llmq::CFinalCommitment::LEGACY_BLS_INDEXED_QUORUM_VERSION};
CHashVerifier<Stream> bls_input{&s};
HashVerifier<Stream> bls_input{s};
bls_input >> CBLSPublicKeyVersionWrapper(commitment.quorumPublicKey, legacy) >> commitment.quorumVvecHash >>
CBLSSignatureVersionWrapper(commitment.quorumSig, legacy) >>
CBLSSignatureVersionWrapper(commitment.membersSig, legacy);
Expand Down
39 changes: 3 additions & 36 deletions src/hash.h
Original file line number Diff line number Diff line change
Expand Up @@ -200,53 +200,20 @@ class HashVerifier : public HashWriter
}
};

template<typename Source>
class CHashVerifier : public CHashWriter
{
private:
Source* source;

public:
explicit CHashVerifier(Source* source_) : CHashWriter(source_->GetType(), source_->GetVersion()), source(source_) {}

void read(Span<std::byte> dst)
{
source->read(dst);
this->write(dst);
}

void ignore(size_t nSize)
{
std::byte data[1024];
while (nSize > 0) {
size_t now = std::min<size_t>(nSize, 1024);
read({data, now});
nSize -= now;
}
}

template<typename T>
CHashVerifier<Source>& operator>>(T&& obj)
{
::Unserialize(*this, obj);
return (*this);
}
};

/** Writes data to an underlying source stream, while hashing the written data. */
template <typename Source>
class HashedSourceWriter : public CHashWriter
class HashedSourceWriter : public HashWriter
{
private:
Source& m_source;

public:
explicit HashedSourceWriter(Source& source LIFETIMEBOUND) : CHashWriter{source.GetType(), source.GetVersion()}, m_source{source} {}
explicit HashedSourceWriter(Source& source LIFETIMEBOUND) : HashWriter{}, m_source{source} {}

void write(Span<const std::byte> src)
{
m_source.write(src);
CHashWriter::write(src);
HashWriter::write(src);
}

template <typename T>
Expand Down
Loading
Loading