Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Serialization improvements step 5 (blockencodings) #18112

Merged
merged 5 commits into from
Mar 5, 2020
+87 −126
Merged

Serialization improvements step 5 (blockencodings) #18112

Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3ca574c
Convert CCompactSize to proper formatter
sipa Feb 4, 2020
56dd9f0
Make VectorFormatter support stateful formatters
ryanofsky Feb 16, 2020
1063339
Add DifferenceFormatter
ryanofsky Feb 16, 2020
e574fff
Add CustomUintFormatter
sipa Feb 16, 2020
353f376
Convert blockencodings.h to new serialization framework
sipa Feb 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Convert blockencodings.h to new serialization framework
  • Loading branch information
@sipa
sipa committed Feb 25, 2020
commit 353f376277ad9b87e03c9ccbc1028c4b6d12e8ea
126 changes: 16 additions & 110 deletions src/blockencodings.h
View file
Original file line number Diff line number Diff line change
@@ -10,20 +10,9 @@

class CTxMemPool;

// Dumb helper to handle CTransaction compression at serialize-time
struct TransactionCompressor {
private:
CTransactionRef& tx;
public:
explicit TransactionCompressor(CTransactionRef& txIn) : tx(txIn) {}

ADD_SERIALIZE_METHODS;

template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITE(tx); //TODO: Compress tx encoding
}
};
// Transaction compression schemes for compact block relay can be introduced by writing
// an actual formatter here.
using TransactionCompression = DefaultFormatter;

class DifferenceFormatter
{
@@ -53,39 +42,9 @@ class BlockTransactionsRequest {
uint256 blockhash;
std::vector<uint16_t> indexes;

ADD_SERIALIZE_METHODS;

template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITE(blockhash);
uint64_t indexes_size = (uint64_t)indexes.size();
READWRITE(COMPACTSIZE(indexes_size));
if (ser_action.ForRead()) {
size_t i = 0;
while (indexes.size() < indexes_size) {
indexes.resize(std::min((uint64_t)(1000 + indexes.size()), indexes_size));
for (; i < indexes.size(); i++) {
uint64_t index = 0;
READWRITE(COMPACTSIZE(index));
if (index > std::numeric_limits<uint16_t>::max())
throw std::ios_base::failure("index overflowed 16 bits");
indexes[i] = index;
}
}

int32_t offset = 0;
for (size_t j = 0; j < indexes.size(); j++) {
if (int32_t(indexes[j]) + offset > std::numeric_limits<uint16_t>::max())
throw std::ios_base::failure("indexes overflowed 16 bits");
indexes[j] = indexes[j] + offset;
offset = int32_t(indexes[j]) + 1;
}
} else {
for (size_t i = 0; i < indexes.size(); i++) {
uint64_t index = indexes[i] - (i == 0 ? 0 : (indexes[i - 1] + 1));
READWRITE(COMPACTSIZE(index));
}
}
SERIALIZE_METHODS(BlockTransactionsRequest, obj)
{
READWRITE(obj.blockhash, Using<VectorFormatter<DifferenceFormatter>>(obj.indexes));
}
};

@@ -99,24 +58,9 @@ class BlockTransactions {
explicit BlockTransactions(const BlockTransactionsRequest& req) :
blockhash(req.blockhash), txn(req.indexes.size()) {}

ADD_SERIALIZE_METHODS;

template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITE(blockhash);
uint64_t txn_size = (uint64_t)txn.size();
READWRITE(COMPACTSIZE(txn_size));
if (ser_action.ForRead()) {
size_t i = 0;
while (txn.size() < txn_size) {
txn.resize(std::min((uint64_t)(1000 + txn.size()), txn_size));
for (; i < txn.size(); i++)
READWRITE(TransactionCompressor(txn[i]));
}
} else {
for (size_t i = 0; i < txn.size(); i++)
READWRITE(TransactionCompressor(txn[i]));
}
SERIALIZE_METHODS(BlockTransactions, obj)
{
READWRITE(obj.blockhash, Using<VectorFormatter<TransactionCompression>>(obj.txn));
}
};

@@ -127,17 +71,7 @@ struct PrefilledTransaction {
uint16_t index;
CTransactionRef tx;

ADD_SERIALIZE_METHODS;

template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
uint64_t idx = index;
READWRITE(COMPACTSIZE(idx));
if (idx > std::numeric_limits<uint16_t>::max())
throw std::ios_base::failure("index overflowed 16-bits");
index = idx;
READWRITE(TransactionCompressor(tx));
}
SERIALIZE_METHODS(PrefilledTransaction, obj) { READWRITE(COMPACTSIZE(obj.index), Using<TransactionCompression>(obj.tx)); }
};

typedef enum ReadStatus_t
@@ -175,43 +109,15 @@ class CBlockHeaderAndShortTxIDs {

size_t BlockTxCount() const { return shorttxids.size() + prefilledtxn.size(); }

ADD_SERIALIZE_METHODS;

template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITE(header);
READWRITE(nonce);

uint64_t shorttxids_size = (uint64_t)shorttxids.size();
READWRITE(COMPACTSIZE(shorttxids_size));
SERIALIZE_METHODS(CBlockHeaderAndShortTxIDs, obj)
{
READWRITE(obj.header, obj.nonce, Using<VectorFormatter<CustomUintFormatter<SHORTTXIDS_LENGTH>>>(obj.shorttxids), obj.prefilledtxn);
if (ser_action.ForRead()) {
size_t i = 0;
while (shorttxids.size() < shorttxids_size) {
shorttxids.resize(std::min((uint64_t)(1000 + shorttxids.size()), shorttxids_size));
for (; i < shorttxids.size(); i++) {
uint32_t lsb = 0; uint16_t msb = 0;
READWRITE(lsb);
READWRITE(msb);
shorttxids[i] = (uint64_t(msb) << 32) | uint64_t(lsb);
static_assert(SHORTTXIDS_LENGTH == 6, "shorttxids serialization assumes 6-byte shorttxids");
}
}
} else {
for (size_t i = 0; i < shorttxids.size(); i++) {
uint32_t lsb = shorttxids[i] & 0xffffffff;
uint16_t msb = (shorttxids[i] >> 32) & 0xffff;
READWRITE(lsb);
READWRITE(msb);
if (obj.BlockTxCount() > std::numeric_limits<uint16_t>::max()) {
throw std::ios_base::failure("indexes overflowed 16 bits");
}
obj.FillShortTxIDSelector();
}

READWRITE(prefilledtxn);

if (BlockTxCount() > std::numeric_limits<uint16_t>::max())
throw std::ios_base::failure("indexes overflowed 16 bits");

if (ser_action.ForRead())
FillShortTxIDSelector();
}
};