net: CAddress & CAddrMan: (un)serialize as ADDRv2

Change the serialization of `CAddrMan` to serialize its addresses in ADDRv2/BIP155 format by default. Introduce a new `CAddrMan` format version (3). Add support for ADDRv2 format in `CAddress` (un)serialization. Co-authored-by: Carl Dong <contact@carldong.me> (cherry picked from commit 201a459)
zingolabs · Oct 13, 2021 · bd2c485 · bd2c485
1 parent e86ce4e
commit bd2c485
Show file tree

Hide file tree

Showing 5 changed files with 166 additions and 16 deletions.
diff --git a/doc/release-notes.md b/doc/release-notes.md
@@ -1,6 +1,14 @@
 (note: this is a temporary file, to be added-to by anybody, and moved to
 release-notes at release time)
 
+The node's known peers are persisted to disk in a file called `peers.dat`. The
+format of this file has been changed in a backwards-incompatible way in order to
+accommodate the storage of Tor v3 and other BIP155 addresses. This means that if
+the file is modified by 0.21.0 or newer then older versions will not be able to
+read it. Those old versions, in the event of a downgrade, will log an error
+message that deserialization has failed and will continue normal operation
+as if the file was missing, creating a new empty one. (#19954)
+
 Notable changes
 ===============
 

diff --git a/src/addrman.h b/src/addrman.h
@@ -8,8 +8,10 @@
 #include "netaddress.h"
 #include "protocol.h"
 #include "random.h"
+#include "streams.h"
 #include "sync.h"
 #include "timedata.h"
+#include "tinyformat.h"
 #include "util.h"
 
 #include <map>
@@ -260,9 +262,14 @@ class CAddrMan
     void Connected_(const CService &addr, int64_t nTime);
 
 public:
+    //! Serialization versions.
+    enum class Format : uint8_t {
+        V0_HISTORICAL = 0,    //!< historic format, before commit e6b343d88
+        V1_BIP155 = 1,        //!< same as V2_ASMAP plus addresses are in BIP155 format
+    };
     /**
      * serialized format:
-     * * version byte (currently 1)
+     * * version byte (@see `Format`)
      * * 0x20 + nKey (serialized as if it were a vector, for backward compatibility)
      * * nNew
      * * nTried
@@ -289,13 +296,16 @@ class CAddrMan
      * We don't use SERIALIZE_METHODS since the serialization and deserialization code has
      * very little in common.
      */
-    template<typename Stream>
-    void Serialize(Stream &s) const
+    template <typename Stream>
+    void Serialize(Stream& s_) const
     {
         LOCK(cs);
 
-        unsigned char nVersion = 1;
-        s << nVersion;
+        // Always serialize in the latest version (currently Format::V3_BIP155).
+
+        OverrideStream<Stream> s(&s_, s_.GetType(), s_.GetVersion() | ADDRV2_FORMAT);
+
+        s << static_cast<uint8_t>(Format::V1_BIP155);
         s << ((unsigned char)32);
         s << nKey;
         s << nNew;
@@ -339,15 +349,35 @@ class CAddrMan
         }
     }
 
-    template<typename Stream>
-    void Unserialize(Stream& s)
+    template <typename Stream>
+    void Unserialize(Stream& s_)
     {
         LOCK(cs);
 
         Clear();
 
-        unsigned char nVersion;
-        s >> nVersion;
+
+        Format format;
+        s_ >> Using<CustomUintFormatter<1>>(format);
+
+        static constexpr Format maximum_supported_format = Format::V3_BIP155;
+        if (format > maximum_supported_format) {
+            throw std::ios_base::failure(strprintf(
+                "Unsupported format of addrman database: %u. Maximum supported is %u. "
+                "Continuing operation without using the saved list of peers.",
+                static_cast<uint8_t>(format),
+                static_cast<uint8_t>(maximum_supported_format)));
+        }
+
+        int stream_version = s_.GetVersion();
+        if (format >= Format::V3_BIP155) {
+            // Add ADDRV2_FORMAT to the version so that the CNetAddr and CAddress
+            // unserialize methods know that an address in addrv2 format is coming.
+            stream_version |= ADDRV2_FORMAT;
+        }
+
+        OverrideStream<Stream> s(&s_, s_.GetType(), stream_version);
+
         unsigned char nKeySize;
         s >> nKeySize;
         if (nKeySize != 32) throw std::ios_base::failure("Incorrect keysize in addrman deserialization");
@@ -356,7 +386,7 @@ class CAddrMan
         s >> nTried;
         int nUBuckets = 0;
         s >> nUBuckets;
-        if (nVersion != 0) {
+        if (format >= Format::V1_BIP155) {
             nUBuckets ^= (1 << 30);
         }
 
@@ -375,7 +405,7 @@ class CAddrMan
             mapAddr[info] = n;
             info.nRandomPos = vRandom.size();
             vRandom.push_back(n);
-            if (nVersion != 1 || nUBuckets != ADDRMAN_NEW_BUCKET_COUNT) {
+            if (format != Format::V0_HISTORICAL || nUBuckets != ADDRMAN_NEW_BUCKET_COUNT) {
                 // In case the new table data cannot be used (nVersion unknown, or bucket count wrong),
                 // immediately try to give them a reference based on their primary source address.
                 int nUBucket = info.GetNewBucket(nKey);

diff --git a/src/protocol.h b/src/protocol.h
@@ -79,10 +79,9 @@ enum ServiceFlags : uint64_t {
 class CAddress : public CService
 {
 public:
-    CAddress();
-    explicit CAddress(CService ipIn, ServiceFlags nServicesIn);
-
-    void Init();
+    CAddress() : CService{} {};
+    CAddress(CService ipIn, ServiceFlags nServicesIn) : CService{ipIn}, nServices{nServicesIn} {};
+    CAddress(CService ipIn, ServiceFlags nServicesIn, uint32_t nTimeIn) : CService{ipIn}, nTime{nTimeIn}, nServices{nServicesIn} {};
 
     SERIALIZE_METHODS(CAddress, obj)
     {
@@ -95,7 +94,14 @@ class CAddress : public CService
             (nVersion >= CADDR_TIME_VERSION && !(s.GetType() & SER_GETHASH))) {
             READWRITE(obj.nTime);
         }
-        READWRITE(Using<CustomUintFormatter<8>>(obj.nServices));
+        if (nVersion & ADDRV2_FORMAT) {
+            uint64_t services_tmp;
+            SER_WRITE(obj, services_tmp = obj.nServices);
+            READWRITE(Using<CompactSizeFormatter<false>>(services_tmp));
+            SER_READ(obj, obj.nServices = static_cast<ServiceFlags>(services_tmp));
+        } else {
+            READWRITE(Using<CustomUintFormatter<8>>(obj.nServices));
+        }
         READWRITEAS(CService, obj);
     }
 

diff --git a/src/streams.h b/src/streams.h
@@ -106,6 +106,7 @@ class OverrideStream
 
     int GetVersion() const { return nVersion; }
     int GetType() const { return nType; }
+    void ignore(size_t size) { return stream->ignore(size); }
 };
 
 template<typename S>

diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp
@@ -2,8 +2,12 @@
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or https://www.opensource.org/licenses/mit-license.php .
 
+<<<<<<< HEAD
 #include "netbase.h"
 #include "test/test_bitcoin.h"
+#include "protocol.h"
+#include "serialize.h"
+#include "streams.h"
 
 #include <string>
 
@@ -355,4 +359,105 @@ BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters)
     BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0example.com\0", 35), ret));
 }
 
+// Since CNetAddr (un)ser is tested separately in net_tests.cpp here we only
+// try a few edge cases for port, service flags and time.
+
+static const std::vector<CAddress> fixture_addresses({
+    CAddress(
+        CService(CNetAddr(in6addr_loopback), 0 /* port */),
+        NODE_NONE,
+        0x4966bc61U /* Fri Jan  9 02:54:25 UTC 2009 */
+    ),
+    CAddress(
+        CService(CNetAddr(in6addr_loopback), 0x00f1 /* port */),
+        NODE_NETWORK,
+        0x83766279U /* Tue Nov 22 11:22:33 UTC 2039 */
+    ),
+    CAddress(
+        CService(CNetAddr(in6addr_loopback), 0xf1f2 /* port */),
+        static_cast<ServiceFlags>(NODE_WITNESS | NODE_COMPACT_FILTERS | NODE_NETWORK_LIMITED),
+        0xffffffffU /* Sun Feb  7 06:28:15 UTC 2106 */
+    )
+});
+
+// fixture_addresses should equal to this when serialized in V1 format.
+// When this is unserialized from V1 format it should equal to fixture_addresses.
+static constexpr const char* stream_addrv1_hex =
+    "03" // number of entries
+
+    "61bc6649"                         // time, Fri Jan  9 02:54:25 UTC 2009
+    "0000000000000000"                 // service flags, NODE_NONE
+    "00000000000000000000000000000001" // address, fixed 16 bytes (IPv4 embedded in IPv6)
+    "0000"                             // port
+
+    "79627683"                         // time, Tue Nov 22 11:22:33 UTC 2039
+    "0100000000000000"                 // service flags, NODE_NETWORK
+    "00000000000000000000000000000001" // address, fixed 16 bytes (IPv6)
+    "00f1"                             // port
+
+    "ffffffff"                         // time, Sun Feb  7 06:28:15 UTC 2106
+    "4804000000000000"                 // service flags, NODE_WITNESS | NODE_COMPACT_FILTERS | NODE_NETWORK_LIMITED
+    "00000000000000000000000000000001" // address, fixed 16 bytes (IPv6)
+    "f1f2";                            // port
+
+// fixture_addresses should equal to this when serialized in V2 format.
+// When this is unserialized from V2 format it should equal to fixture_addresses.
+static constexpr const char* stream_addrv2_hex =
+    "03" // number of entries
+
+    "61bc6649"                         // time, Fri Jan  9 02:54:25 UTC 2009
+    "00"                               // service flags, COMPACTSIZE(NODE_NONE)
+    "02"                               // network id, IPv6
+    "10"                               // address length, COMPACTSIZE(16)
+    "00000000000000000000000000000001" // address
+    "0000"                             // port
+
+    "79627683"                         // time, Tue Nov 22 11:22:33 UTC 2039
+    "01"                               // service flags, COMPACTSIZE(NODE_NETWORK)
+    "02"                               // network id, IPv6
+    "10"                               // address length, COMPACTSIZE(16)
+    "00000000000000000000000000000001" // address
+    "00f1"                             // port
+
+    "ffffffff"                         // time, Sun Feb  7 06:28:15 UTC 2106
+    "fd4804"                           // service flags, COMPACTSIZE(NODE_WITNESS | NODE_COMPACT_FILTERS | NODE_NETWORK_LIMITED)
+    "02"                               // network id, IPv6
+    "10"                               // address length, COMPACTSIZE(16)
+    "00000000000000000000000000000001" // address
+    "f1f2";                            // port
+
+BOOST_AUTO_TEST_CASE(caddress_serialize_v1)
+{
+    CDataStream s(SER_NETWORK, PROTOCOL_VERSION);
+
+    s << fixture_addresses;
+    BOOST_CHECK_EQUAL(HexStr(s), stream_addrv1_hex);
+}
+
+BOOST_AUTO_TEST_CASE(caddress_unserialize_v1)
+{
+    CDataStream s(ParseHex(stream_addrv1_hex), SER_NETWORK, PROTOCOL_VERSION);
+    std::vector<CAddress> addresses_unserialized;
+
+    s >> addresses_unserialized;
+    BOOST_CHECK(fixture_addresses == addresses_unserialized);
+}
+
+BOOST_AUTO_TEST_CASE(caddress_serialize_v2)
+{
+    CDataStream s(SER_NETWORK, PROTOCOL_VERSION | ADDRV2_FORMAT);
+
+    s << fixture_addresses;
+    BOOST_CHECK_EQUAL(HexStr(s), stream_addrv2_hex);
+}
+
+BOOST_AUTO_TEST_CASE(caddress_unserialize_v2)
+{
+    CDataStream s(ParseHex(stream_addrv2_hex), SER_NETWORK, PROTOCOL_VERSION | ADDRV2_FORMAT);
+    std::vector<CAddress> addresses_unserialized;
+
+    s >> addresses_unserialized;
+    BOOST_CHECK(fixture_addresses == addresses_unserialized);
+}
+
 BOOST_AUTO_TEST_SUITE_END()