From 097383cd440f8fce1dd6a17b3222ad07921a47b7 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Tue, 3 Dec 2024 20:41:31 +0800 Subject: [PATCH 01/11] feature: custom confing for en-/decoding and jam-codec compatibility Signed-off-by: Dmitriy Khaustov aka xDimon --- CMakeLists.txt | 14 +- include/scale/detail/compact_integer.hpp | 199 +++++++++++ include/scale/detail/jam_compact_integer.hpp | 126 +++++++ include/scale/scale_decoder_stream.hpp | 55 +++- include/scale/scale_encoder_stream.hpp | 55 +++- include/scale/scale_error.hpp | 4 +- include/scale/tune.hpp | 21 ++ include/scale/types.hpp | 16 - src/compact_len_utils.hpp | 42 +-- src/encode_append.cpp | 7 +- src/scale_decoder_stream.cpp | 89 +---- src/scale_encoder_stream.cpp | 107 +----- src/scale_error.cpp | 4 + test/CMakeLists.txt | 9 + test/scale_collection_test.cpp | 329 +++++++++---------- test/scale_compact_test.cpp | 100 +++++- test/scale_strings_test.cpp | 39 ++- test/scale_tune_test.cpp | 81 +++++ 18 files changed, 873 insertions(+), 424 deletions(-) create mode 100644 include/scale/detail/compact_integer.hpp create mode 100644 include/scale/detail/jam_compact_integer.hpp create mode 100644 include/scale/tune.hpp create mode 100644 test/scale_tune_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index eb81804..4f5c022 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,8 +13,8 @@ endif() include(${CMAKE_CURRENT_LIST_DIR}/cmake/HunterGate.cmake) HunterGate( - URL https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm21.zip - SHA1 5b52ab9a309771f172ca609a46e26dde60a8edd7 + URL https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm25.tar.gz + SHA1 bf5742041306c4b2c8b65b9c2d2af712a36ac3f9 ) project(Scale LANGUAGES CXX VERSION 1.1.0) @@ -25,6 +25,9 @@ set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +option(JAM_COMPATIBLE "Build compatible with JAM-codec" OFF) +option(CUSTOM_CONFIG_SUPPORT "Support custom config of streams" OFF) + option(BUILD_TESTS "Whether to include the test suite in build" OFF) hunter_add_package(Boost) @@ -33,6 +36,13 @@ find_package(Boost CONFIG REQUIRED) hunter_add_package(qtils) find_package(qtils CONFIG REQUIRED) +if (JAM_COMPATIBLE) + add_compile_definitions(JAM_COMPATIBILITY_ENABLED) +endif () +if (CUSTOM_CONFIG_SUPPORT) + add_compile_definitions(CUSTOM_CONFIG_ENABLED) +endif () + add_subdirectory(src) if (BUILD_TESTS) diff --git a/include/scale/detail/compact_integer.hpp b/include/scale/detail/compact_integer.hpp new file mode 100644 index 0000000..5a4fbdf --- /dev/null +++ b/include/scale/detail/compact_integer.hpp @@ -0,0 +1,199 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +namespace scale::detail { + + /// min integer encoded by 2 bytes + constexpr static size_t kMinUint16 = (1ul << 6u); + /// min integer encoded by 4 bytes + constexpr static size_t kMinUint32 = (1ul << 14u); + /// min integer encoded as multibyte + constexpr static size_t kMinBigInteger = (1ul << 30u); + + /// Returns the compact encoded length for the given value. + size_t compactLen(std::unsigned_integral auto val) { + if (val < kMinUint16) return 1; + if (val < kMinUint32) return 2; + if (val < kMinBigInteger) return 4; + size_t counter = 1; + while ((val >>= 8)) ++counter; + return counter; + } + + /** + * Encodes any integer type to compact-integer representation + * @tparam T integer type + * @tparam S output stream type + * @param value integer value + * @return byte array representation of value as compact-integer + */ + template + requires(std::integral or std::is_same_v) + void encodeCompactInteger(T integer, S &s) { + boost::multiprecision::cpp_int value{integer}; + + // cannot encode negative numbers + // there is no description how to encode compact negative numbers + if (value < 0) { + raise(EncodeError::NEGATIVE_COMPACT_INTEGER); + } + + if (value < kMinUint16) { + uint8_t v = (value.convert_to() << 2u) | 0b00; + return encodeInteger(v, s); + } + + else if (value < kMinUint32) { + // only values from [kMinUint16, kMinUint32) can be put here + uint16_t v = (value.convert_to() << 2u) | 0b01; + return encodeInteger(v, s); + } + + else if (value < kMinBigInteger) { + // only values from [kMinUint32, kMinBigInteger) can be put here + uint32_t v = (value.convert_to() << 2u) | 0b10; + return encodeInteger(v, s); + } + + // number of bytes required to represent value + size_t significant_bytes_n = msb(value) / 8 + 1; + + if (significant_bytes_n > 67) { + raise(EncodeError::COMPACT_INTEGER_TOO_BIG); + } + + // The upper 6 bits of the header are used to encode the number of bytes + // required to store the big integer. The value stored in these 6 bits + // ranges from 0 to 63 (2^6 - 1). However, the actual byte count starts + // from 4, so we subtract 4 from the byte count before encoding it. + // This makes the range of byte counts for storing big integers 4 to 67. + // To construct the final header, the upper 6 bits are shifted left by + // 2 positions (equivalent to multiplying by 4). + // The lower 2 bits (minor bits) store the encoding option, which in this + // case is 0b11 (decimal value 3). The final header is formed by adding 3 + // to the result of the previous operations. + uint8_t header = ((significant_bytes_n - 4) << 2u) | 0b11; + + s << header; + + for (auto v = value; v != 0; v >>= 8) { + // push back the least significant byte + s << static_cast(v & 0xff); + } + } + + template + requires std::is_same_v + T decodeCompactInteger(S &stream) { + auto first_byte = stream.nextByte(); + + const uint8_t flag = (first_byte)&0b00000011u; + + size_t number = 0u; + + switch (flag) { + case 0b00u: { + number = static_cast(first_byte >> 2u); + break; + } + + case 0b01u: { + auto second_byte = stream.nextByte(); + + number = (static_cast((first_byte)&0b11111100u) + + static_cast(second_byte) * 256u) + >> 2u; + if ((number >> 6) == 0) { + raise(DecodeError::REDUNDANT_COMPACT_ENCODING); + } + break; + } + + case 0b10u: { + number = first_byte; + size_t multiplier = 256u; + if (!stream.hasMore(3u)) { + raise(DecodeError::NOT_ENOUGH_DATA); + } + + for (auto i = 0u; i < 3u; ++i) { + // we assured that there are 3 more bytes, + // no need to make checks in a loop + number += (stream.nextByte()) * multiplier; + multiplier = multiplier << 8u; + } + number = number >> 2u; + if ((number >> 14) == 0) { + raise(DecodeError::REDUNDANT_COMPACT_ENCODING); + } + break; + } + + case 0b11: { + auto bytes_count = ((first_byte) >> 2u) + 4u; + if (!stream.hasMore(bytes_count)) { + raise(DecodeError::NOT_ENOUGH_DATA); + } + + CompactInteger multiplier{1u}; + CompactInteger value = 0; + // we assured that there are m more bytes, + // no need to make checks in a loop + for (auto i = 0u; i < bytes_count; ++i) { + value += (stream.nextByte()) * multiplier; + multiplier *= 256u; + } + if (value.is_zero()) { + raise(DecodeError::REDUNDANT_COMPACT_ENCODING); + } + auto bits = msb(value) + 1; + if (bits <= 30 or (bits + 7) / 8 < bytes_count) { + raise(DecodeError::REDUNDANT_COMPACT_ENCODING); + } + return value; // special case + } + + default: + UNREACHABLE + } + + return CompactInteger{number}; + } + + /** + * Decodes any integer type from compact-integer representation + * @tparam T integer type + * @tparam S input stream type + * @param value integer value + * @return value according compact-integer representation + */ + template + requires std::unsigned_integral + T decodeCompactInteger(S &s) { + auto integer = decodeCompactInteger(s); + if (not integer.is_zero() + and msb(integer) >= std::numeric_limits::digits) { + raise(DecodeError::DECODED_VALUE_OVERFLOWS_TARGET); + } + return static_cast(integer); + } + + +} // namespace scale::detail diff --git a/include/scale/detail/jam_compact_integer.hpp b/include/scale/detail/jam_compact_integer.hpp new file mode 100644 index 0000000..cb73456 --- /dev/null +++ b/include/scale/detail/jam_compact_integer.hpp @@ -0,0 +1,126 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +namespace scale::detail { + + /** + * Encodes any integer type to jam-compact-integer representation + * @tparam T integer type + * @tparam S output stream type + * @param value integer value + * @return byte array representation of value as jam-compact-integer + */ + template + requires(std::unsigned_integral or std::is_same_v) + void encodeJamCompactInteger(T integer, S &s) { + size_t value; + + if constexpr (std::is_same_v) { + // cannot encode negative numbers + // there is no description how to encode compact negative numbers + if (integer < 0) { + raise(EncodeError::NEGATIVE_COMPACT_INTEGER); + } + if (integer.is_zero()) { + value = 0; + } else { + if (msb(integer) >= std::numeric_limits::digits) { + raise(EncodeError::VALUE_TOO_BIG_FOR_COMPACT_REPRESENTATION); + } + value = integer.template convert_to(); + } + } else { + value = static_cast(integer); + } + + if (value < 0x80) { + s << static_cast(value); + return; + } + + std::array bytes; + uint8_t &prefix = bytes[0] = 0; + size_t len = 1; + + for (decltype(value) i = value; i != 0; i >>= 8) { + // minimal value in prefix + if (i <= (static_cast(~prefix) >> 1)) { + prefix |= i; + break; + } + prefix = (prefix >> 1) | 0x80; + bytes[len++] = static_cast(i & 0xff); + } + + for (auto byte : bytes) { + s << byte; + if (--len == 0) break; + } + } + + /** + * Decodes any integer type from jam-compact-integer representation + * @tparam T integer type + * @tparam S input stream type + * @param value integer value + * @return value according jam-compact-integer representation + */ + template + requires std::unsigned_integral or std::is_same_v + T decodeJamCompactInteger(auto &s) { + uint8_t byte; + + s >> byte; + + if (byte == 0) { + return 0; + } + + uint8_t len_bits = byte; + uint8_t val_bits = byte; + uint8_t val_mask = 0xff; + + size_t value = 0; + + for (uint8_t i = 0; i < 8; ++i) { + val_mask >>= 1; + val_bits &= val_mask; + + if ((len_bits & static_cast(0x80)) == 0) { // no more significant bytes + value |= static_cast(val_bits) << (8 * i); + break; + } + len_bits <<= 1; + + s >> byte; + value |= static_cast(byte) << (8 * i); + } + if (val_bits == 0 and (byte & ~val_mask) == 0) { + raise(DecodeError::REDUNDANT_COMPACT_ENCODING); + } + + if constexpr (not std::is_same_v + and not std::is_same_v) { + if (value > std::numeric_limits::max()) { + raise(DecodeError::DECODED_VALUE_OVERFLOWS_TARGET); + } + } + + return static_cast(value); + } + +} // namespace scale::detail diff --git a/include/scale/scale_decoder_stream.hpp b/include/scale/scale_decoder_stream.hpp index ad3c962..492dd97 100644 --- a/include/scale/scale_decoder_stream.hpp +++ b/include/scale/scale_decoder_stream.hpp @@ -6,6 +6,9 @@ #pragma once +#ifdef CUSTOM_CONFIG_ENABLED +#include +#endif #include #include #include @@ -14,7 +17,9 @@ #include #include +#include #include +#include #include #include #include @@ -25,14 +30,28 @@ namespace scale { // special tag to differentiate decoding streams from others static constexpr auto is_decoder_stream = true; - explicit ScaleDecoderStream(ConstSpanOfBytes data) - : span_{data}, current_index_{0} {} + explicit ScaleDecoderStream(ConstSpanOfBytes data) : span_{data} {} + +#ifdef CUSTOM_CONFIG_ENABLED + template + requires(std::is_class_v and not std::is_union_v) + ScaleDecoderStream(ConstSpanOfBytes data, const ConfigT &config) + : span_{data}, config_(std::cref(config)) {} +#else + template + requires(std::is_class_v and not std::is_union_v) + [[deprecated("Scale has compiled without custom config support")]] // + ScaleDecoderStream(ConstSpanOfBytes data, const ConfigT &config) = delete; +#endif template T decodeCompact() { - // TODO(turuslan): don't allocate cpp_int - scale::CompactInteger big; - *this >> big; + scale::CompactInteger big = +#ifdef JAM_COMPATIBILITY_ENABLED + detail::decodeJamCompactInteger(*this); +#else + detail::decodeCompactInteger(*this); +#endif if (not big.is_zero() and msb(big) >= std::numeric_limits::digits) { raise(DecodeError::TOO_MANY_ITEMS); } @@ -345,6 +364,25 @@ namespace scale { return current_index_; } +#ifdef CUSTOM_CONFIG_ENABLED + template + const T &getConfig() const { + if (not config_.has_value()) { + throw std::runtime_error("Stream created without any custom config"); + } + if (config_.type().hash_code() + != typeid(std::reference_wrapper).hash_code()) { + throw std::runtime_error("Stream created with other custom config"); + } + return std::any_cast>(config_).get(); + } +#else + template + [[deprecated("Scale has compiled without custom config support")]] // + const T & + getConfig_() const = delete; +#endif + private: bool decodeBool(); /** @@ -378,7 +416,12 @@ namespace scale { } ByteSpan span_; - SizeType current_index_; + +#ifdef CUSTOM_CONFIG_ENABLED + const std::any config_{}; +#endif + + SizeType current_index_{0}; }; } // namespace scale diff --git a/include/scale/scale_encoder_stream.hpp b/include/scale/scale_encoder_stream.hpp index 3d36e5b..3e1c122 100644 --- a/include/scale/scale_encoder_stream.hpp +++ b/include/scale/scale_encoder_stream.hpp @@ -6,6 +6,9 @@ #pragma once +#ifdef CUSTOM_CONFIG_ENABLED +#include +#endif #include #include #include @@ -13,8 +16,11 @@ #include #include +#include #include +#include #include +#include #include namespace scale { @@ -36,6 +42,28 @@ namespace scale { */ explicit ScaleEncoderStream(bool drop_data); +#ifdef CUSTOM_CONFIG_ENABLED + template + requires(std::is_class_v and not std::is_union_v) + explicit ScaleEncoderStream(const ConfigT &config) + : config_(std::cref(config)) {} + + template + requires(std::is_class_v and not std::is_union_v) + ScaleEncoderStream(const ConfigT &config, bool drop_data) + : config_(std::cref(config)), drop_data_(drop_data) {} +#else + template + requires(std::is_class_v and not std::is_union_v) + [[deprecated("Scale has compiled without custom config support")]] // + ScaleEncoderStream(const ConfigT &config) = delete; + + template + requires(std::is_class_v and not std::is_union_v) + [[deprecated("Scale has compiled without custom config support")]] // + ScaleEncoderStream(const ConfigT &config, bool drop_data) = delete; +#endif + /** * @return vector of bytes containing encoded data */ @@ -225,6 +253,25 @@ namespace scale { */ ScaleEncoderStream &operator<<(const CompactInteger &v); +#ifdef CUSTOM_CONFIG_ENABLED + template + const T &getConfig() const { + if (not config_.has_value()) { + throw std::runtime_error("Stream created without any custom config"); + } + if (config_.type().hash_code() + != typeid(std::reference_wrapper).hash_code()) { + throw std::runtime_error("Stream created with other custom config"); + } + return std::any_cast>(config_).get(); + } +#else + template + [[deprecated("Scale has compiled without custom config support")]] // + const T & + getConfig_() const = delete; +#endif + protected: template void encodeElementOfTuple(const std::tuple &v) { @@ -283,9 +330,13 @@ namespace scale { private: ScaleEncoderStream &encodeOptionalBool(const std::optional &v); - const bool drop_data_; +#ifdef CUSTOM_CONFIG_ENABLED + const std::any config_{}; +#endif + + const bool drop_data_ = false; std::deque stream_; - size_t bytes_written_; + size_t bytes_written_ = 0; }; /** diff --git a/include/scale/scale_error.hpp b/include/scale/scale_error.hpp index d509129..41f3c16 100644 --- a/include/scale/scale_error.hpp +++ b/include/scale/scale_error.hpp @@ -17,6 +17,7 @@ namespace scale { COMPACT_INTEGER_TOO_BIG = 1, ///< compact integer can't be more than 2**536 NEGATIVE_COMPACT_INTEGER, ///< cannot compact-encode negative integers DEREF_NULLPOINTER, ///< dereferencing a null pointer + VALUE_TOO_BIG_FOR_COMPACT_REPRESENTATION, ///< value too bit for compact representation }; /** @@ -28,7 +29,8 @@ namespace scale { TOO_MANY_ITEMS, ///< too many items, cannot address them in memory WRONG_TYPE_INDEX, ///< wrong type index, cannot decode variant INVALID_ENUM_VALUE, ///< enum value which doesn't belong to the enum - REDUNDANT_COMPACT_ENCODING, ///< redundant bytes in compact encoding + REDUNDANT_COMPACT_ENCODING, ///< redundant bytes in compact encoding + DECODED_VALUE_OVERFLOWS_TARGET, ///< encoded value overflows target type }; } // namespace scale diff --git a/include/scale/tune.hpp b/include/scale/tune.hpp new file mode 100644 index 0000000..7473399 --- /dev/null +++ b/include/scale/tune.hpp @@ -0,0 +1,21 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +namespace scale::detail { + + enum Compatibility { + Classic, + Jam, + }; +} + +namespace scale::tune { + inline static constexpr detail::Compatibility classic = + detail::Compatibility::Classic; + inline static constexpr detail::Compatibility jam_compatible = + detail::Compatibility::Jam; + +} // namespace scale::tune diff --git a/include/scale/types.hpp b/include/scale/types.hpp index 81ee64d..543adc9 100644 --- a/include/scale/types.hpp +++ b/include/scale/types.hpp @@ -106,19 +106,3 @@ namespace scale { and HasEmplaceMethod; } // namespace scale - -namespace scale::compact { - - /** - * @brief categories of compact encoding - */ - struct EncodingCategoryLimits { - // min integer encoded by 2 bytes - constexpr static size_t kMinUint16 = (1ul << 6u); - // min integer encoded by 4 bytes - constexpr static size_t kMinUint32 = (1ul << 14u); - // min integer encoded as multibyte - constexpr static size_t kMinBigInteger = (1ul << 30u); - }; - -} // namespace scale::compact diff --git a/src/compact_len_utils.hpp b/src/compact_len_utils.hpp index 5e6c3f0..f477e33 100644 --- a/src/compact_len_utils.hpp +++ b/src/compact_len_utils.hpp @@ -9,25 +9,25 @@ #include namespace scale::compact { - // calculate number of bytes required - inline size_t countBytes(CompactInteger v) { - size_t counter = 0; - do { - ++counter; - } while ((v >>= 8) != 0); - return counter; - } - - /** - * Returns the compact encoded length for the given value. - */ - template , - typename = std::enable_if_t::value>> - uint32_t compactLen(T val) { - if (val < EncodingCategoryLimits::kMinUint16) return 1; - if (val < EncodingCategoryLimits::kMinUint32) return 2; - if (val < EncodingCategoryLimits::kMinBigInteger) return 4; - return countBytes(val); - } +// // calculate number of bytes required +// inline size_t countBytes(CompactInteger v) { +// size_t counter = 0; +// do { +// ++counter; +// } while ((v >>= 8) != 0); +// return counter; +// } +// +// /** +// * Returns the compact encoded length for the given value. +// */ +// template , +// typename = std::enable_if_t::value>> +// uint32_t compactLen(T val) { +// if (val < EncodingCategoryLimits::kMinUint16) return 1; +// if (val < EncodingCategoryLimits::kMinUint32) return 2; +// if (val < EncodingCategoryLimits::kMinBigInteger) return 4; +// return countBytes(val); +// } } // namespace scale::compact diff --git a/src/encode_append.cpp b/src/encode_append.cpp index 09b7254..94dac3f 100644 --- a/src/encode_append.cpp +++ b/src/encode_append.cpp @@ -7,7 +7,8 @@ #include #include -#include "compact_len_utils.hpp" +#include +#include namespace scale { @@ -24,8 +25,8 @@ namespace scale { const std::vector &data) { OUTCOME_TRY(len, scale::decode(data)); auto new_len = (len + 1).convert_to(); - auto encoded_len = compact::compactLen(len.convert_to()); - auto encoded_new_len = compact::compactLen(new_len); + auto encoded_len = detail::compactLen(len.convert_to()); + auto encoded_new_len = detail::compactLen(new_len); return std::make_tuple(new_len, encoded_len, encoded_new_len); } diff --git a/src/scale_decoder_stream.cpp b/src/scale_decoder_stream.cpp index 16b1694..5506ff1 100644 --- a/src/scale_decoder_stream.cpp +++ b/src/scale_decoder_stream.cpp @@ -6,87 +6,16 @@ #include -namespace scale { - namespace { - CompactInteger decodeCompactInteger(ScaleDecoderStream &stream) { - auto first_byte = stream.nextByte(); - - const uint8_t flag = (first_byte)&0b00000011u; - - size_t number = 0u; - - switch (flag) { - case 0b00u: { - number = static_cast(first_byte >> 2u); - break; - } - - case 0b01u: { - auto second_byte = stream.nextByte(); - - number = (static_cast((first_byte)&0b11111100u) - + static_cast(second_byte) * 256u) - >> 2u; - if ((number >> 6) == 0) { - raise(DecodeError::REDUNDANT_COMPACT_ENCODING); - } - break; - } - - case 0b10u: { - number = first_byte; - size_t multiplier = 256u; - if (!stream.hasMore(3u)) { - raise(DecodeError::NOT_ENOUGH_DATA); - } - - for (auto i = 0u; i < 3u; ++i) { - // we assured that there are 3 more bytes, - // no need to make checks in a loop - number += (stream.nextByte()) * multiplier; - multiplier = multiplier << 8u; - } - number = number >> 2u; - if ((number >> 14) == 0) { - raise(DecodeError::REDUNDANT_COMPACT_ENCODING); - } - break; - } - - case 0b11: { - auto bytes_count = ((first_byte) >> 2u) + 4u; - if (!stream.hasMore(bytes_count)) { - raise(DecodeError::NOT_ENOUGH_DATA); - } - - CompactInteger multiplier{1u}; - CompactInteger value = 0; - // we assured that there are m more bytes, - // no need to make checks in a loop - for (auto i = 0u; i < bytes_count; ++i) { - value += (stream.nextByte()) * multiplier; - multiplier *= 256u; - } - if (value.is_zero()) { - raise(DecodeError::REDUNDANT_COMPACT_ENCODING); - } - auto bits = msb(value) + 1; - if (bits <= 30 or (bits + 7) / 8 < bytes_count) { - raise(DecodeError::REDUNDANT_COMPACT_ENCODING); - } - return value; // special case - } - - default: - UNREACHABLE - } - - return CompactInteger{number}; - } - } // namespace +#include +#include +namespace scale { size_t ScaleDecoderStream::decodeLength() { - size_t size = decodeCompact(); +#ifdef JAM_COMPATIBILITY_ENABLED + size_t size = detail::decodeJamCompactInteger(*this); +#else + size_t size = detail::decodeCompactInteger(*this); +#endif if (not hasMore(size)) { raise(DecodeError::NOT_ENOUGH_DATA); } @@ -119,7 +48,7 @@ namespace scale { } ScaleDecoderStream &ScaleDecoderStream::operator>>(CompactInteger &v) { - v = decodeCompactInteger(*this); + v = decodeCompact(); return *this; } diff --git a/src/scale_encoder_stream.cpp b/src/scale_encoder_stream.cpp index 0556d83..4f7be77 100644 --- a/src/scale_encoder_stream.cpp +++ b/src/scale_encoder_stream.cpp @@ -6,104 +6,10 @@ #include -#include "compact_len_utils.hpp" +#include +#include namespace scale { - namespace { - // must not use these functions outside encodeInteger - void encodeFirstCategory(uint8_t value, ScaleEncoderStream &out) { - // only values from [0, kMinUint16) can be put here - out << static_cast(value << 2u); - } - - void encodeSecondCategory(uint16_t value, ScaleEncoderStream &out) { - // only values from [kMinUint16, kMinUint32) can be put here - auto v = value; - v <<= 2u; // v *= 4 - v += 1u; // set 0b01 flag - auto minor_byte = static_cast(v & 0xFFu); - v >>= 8u; - auto major_byte = static_cast(v & 0xFFu); - - out << minor_byte << major_byte; - } - - void encodeThirdCategory(uint32_t value, ScaleEncoderStream &out) { - // only values from [kMinUint32, kMinBigInteger) can be put here - uint32_t v = (value << 2u) + 2; - scale::detail::encodeInteger(v, out); - } - - /** - * @brief compact-encodes CompactInteger - * @param value source CompactInteger value - */ - void encodeCompactInteger(const CompactInteger &value, - ScaleEncoderStream &out) { - // cannot encode negative numbers - // there is no description how to encode compact negative numbers - if (value < 0) { - raise(EncodeError::NEGATIVE_COMPACT_INTEGER); - } - - if (value < compact::EncodingCategoryLimits::kMinUint16) { - encodeFirstCategory(value.convert_to(), out); - return; - } - - if (value < compact::EncodingCategoryLimits::kMinUint32) { - encodeSecondCategory(value.convert_to(), out); - return; - } - - if (value < compact::EncodingCategoryLimits::kMinBigInteger) { - encodeThirdCategory(value.convert_to(), out); - return; - } - - // number of bytes required to represent value - size_t bigIntLength = compact::countBytes(value); - - // number of bytes to scale-encode value - // 1 byte is reserved for header - size_t requiredLength = 1 + bigIntLength; - - if (bigIntLength > 67) { - raise(EncodeError::COMPACT_INTEGER_TOO_BIG); - } - - ByteArray result; - result.reserve(requiredLength); - - /* The value stored in 6 major bits of header is used - * to encode number of bytes for storing big integer. - * Value formed by 6 bits varies from 0 to 63 == 2^6 - 1, - * However big integer byte count starts from 4, - * so to store this number we should decrease this value by 4. - * And the range of bytes number for storing big integer - * becomes 4 .. 67. To form resulting header we need to move - * those bits representing bytes count to the left by 2 positions - * by means of multiplying by 4. - * Minor 2 bits store encoding option, in our case it is 0b11 == 3 - * We just add 3 to the result of operations above - */ - uint8_t header = (bigIntLength - 4) * 4 + 3; - - result.push_back(header); - - CompactInteger v{value}; - for (size_t i = 0; i < bigIntLength; ++i) { - result.push_back(static_cast( - v & 0xFF)); // push back least significant byte - v >>= 8; - } - - for (const uint8_t c : result) { - out << c; - } - } - } // namespace - ScaleEncoderStream::ScaleEncoderStream() : drop_data_{false}, bytes_written_{0} {} @@ -152,10 +58,17 @@ namespace scale { return *this; } +#ifdef JAM_COMPATIBILITY_ENABLED + ScaleEncoderStream &ScaleEncoderStream::operator<<(const CompactInteger &v) { + detail::encodeJamCompactInteger(v, *this); + return *this; + } +#else ScaleEncoderStream &ScaleEncoderStream::operator<<(const CompactInteger &v) { - encodeCompactInteger(v, *this); + detail::encodeCompactInteger(v, *this); return *this; } +#endif ScaleEncoderStream &ScaleEncoderStream::encodeOptionalBool( const std::optional &v) { diff --git a/src/scale_error.cpp b/src/scale_error.cpp index 09d56bc..0694e2f 100644 --- a/src/scale_error.cpp +++ b/src/scale_error.cpp @@ -11,6 +11,8 @@ OUTCOME_CPP_DEFINE_CATEGORY(scale, EncodeError, e) { switch (e) { case EncodeError::NEGATIVE_COMPACT_INTEGER: return "SCALE encode: compact integers cannot be negative"; + case EncodeError::VALUE_TOO_BIG_FOR_COMPACT_REPRESENTATION: + return "SCALE decode: value too big for compact representation"; case EncodeError::COMPACT_INTEGER_TOO_BIG: return "SCALE encode: compact integers too big"; case EncodeError::DEREF_NULLPOINTER: @@ -35,6 +37,8 @@ OUTCOME_CPP_DEFINE_CATEGORY(scale, DecodeError, e) { return "SCALE decode: decoded enum value does not belong to the enum"; case DecodeError::REDUNDANT_COMPACT_ENCODING: return "SCALE decode: redundant bytes in compact encoding"; + case DecodeError::DECODED_VALUE_OVERFLOWS_TARGET: + return "SCALE decode: encoded value overflows target type"; } return "unknown SCALE DecodeError"; } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8f0aab3..1be0a6d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -151,3 +151,12 @@ addtest(scale_encode_counter_test target_link_libraries(scale_encode_counter_test scale ) + +if (CUSTOM_CONFIG_SUPPORT) + addtest(scale_tune_test + scale_tune_test.cpp + ) + target_link_libraries(scale_tune_test + scale + ) +endif () diff --git a/test/scale_collection_test.cpp b/test/scale_collection_test.cpp index e3c9c07..3989030 100644 --- a/test/scale_collection_test.cpp +++ b/test/scale_collection_test.cpp @@ -16,21 +16,37 @@ using scale::encode; using scale::ScaleDecoderStream; using scale::ScaleEncoderStream; +auto encodeLen = [](size_t i) { + ScaleEncoderStream s; +#ifdef JAM_COMPATIBILITY_ENABLED + scale::detail::encodeJamCompactInteger(i, s); +#else + scale::detail::encodeCompactInteger(i, s); +#endif + return s.to_vector(); +}; + /** * @given collection of 80 items of type uint8_t * @when encodeCollection is applied * @then expected result is obtained: header is 2 byte, items are 1 byte each */ -TEST(Scale, encodeCollectionOf80) { - // 80 items of value 1 - ByteArray collection(80, 1); - auto match = ByteArray{65, 1}; // header - match.insert(match.end(), collection.begin(), collection.end()); - ScaleEncoderStream s; - ASSERT_NO_THROW((s << collection)); - auto &&out = s.to_vector(); - ASSERT_EQ(out.size(), 82); - ASSERT_EQ(out, match); +TEST(CollectionTest, encodeCollectionOf80) { + for (size_t length = 60; length <= 130; ++length) { + ByteArray collection(length); + collection.reserve(length); + for (auto i = 0; i < length; ++i) { + collection.push_back(i % 256); + } + ScaleEncoderStream s; + ASSERT_NO_THROW((s << collection)); + auto &&out = s.to_vector(); + + auto match = encodeLen(collection.size()); // header + match.insert(match.end(), collection.begin(), collection.end()); + + ASSERT_EQ(out, match); + } } /** @@ -38,7 +54,7 @@ TEST(Scale, encodeCollectionOf80) { * @when encodeCollection is applied * @then expected result is obtained: header is 2 byte, items are 1 byte each */ -TEST(Scale, encodeVectorOfBool) { +TEST(CollectionTest, encodeVectorOfBool) { std::vector collection = {true, false, true, false, false, false}; ScaleEncoderStream s; ASSERT_NO_THROW((s << collection)); @@ -47,28 +63,47 @@ TEST(Scale, encodeVectorOfBool) { auto stream = ScaleDecoderStream(out); std::vector decoded; stream >> decoded; - ASSERT_TRUE(std::equal( - decoded.begin(), decoded.end(), collection.begin(), collection.end())); - - // clang-format off - ASSERT_EQ(out, - (ByteArray{ - 24, // header - 1, // first item - 0, // second item - 1, // third item - 0, // fourth item - 0, // fifth item - 0 // sixths item - })); - // clang-format on + ASSERT_TRUE(std::ranges::equal(decoded, collection)); + + ByteArray data{ + // clang-format off + 1, // first item + 0, // second item + 1, // third item + 0, // fourth item + 0, // fifth item + 0, // sixths item + // clang-format on + }; + auto match = encodeLen(collection.size()); + match.insert(match.end(), data.begin(), data.end()); + + ASSERT_TRUE(std::ranges::equal(out, match)); } -TEST(Scale, encodeBitVec) { - auto v = BitVec{{true, true, false, false, false, false, true}}; - auto encoded = ByteArray{(7 << 2), 0b01000011}; - ASSERT_EQ(encode(v).value(), encoded); - ASSERT_EQ(decode(encoded).value(), v); +TEST(CollectionTest, encodeBitVec) { + auto collection = BitVec{{ + // clang-format off + true, true, false, false, false, false, true, false, // 01000011 + false, true, true, false, false // ___00110 + // clang-format on + }}; + ByteArray vector_representation = {0b01000011, 0b00110}; + + ScaleEncoderStream s; + ASSERT_NO_THROW((s << collection)); + auto &&encoded = s.to_vector(); + + auto sizeLen = encodeLen(collection.bits.size()).size(); + auto out = + std::span(std::next(encoded.data(), sizeLen), encoded.size() - sizeLen); + + ASSERT_TRUE(std::ranges::equal(out, vector_representation)); + + auto stream = ScaleDecoderStream(encoded); + BitVec decoded; + stream >> decoded; + ASSERT_TRUE(std::ranges::equal(decoded.bits, collection.bits)); } /** @@ -76,7 +111,7 @@ TEST(Scale, encodeBitVec) { * @when encodeCollection is applied * @then expected result is obtained */ -TEST(Scale, encodeCollectionUint16) { +TEST(CollectionTest, encodeCollectionUint16) { std::vector collection = {1, 2, 3, 4}; ScaleEncoderStream s; ASSERT_NO_THROW((s << collection)); @@ -85,19 +120,20 @@ TEST(Scale, encodeCollectionUint16) { auto stream = ScaleDecoderStream(out); std::vector decoded; stream >> decoded; - ASSERT_TRUE(std::equal( - decoded.begin(), decoded.end(), collection.begin(), collection.end())); - - // clang-format off - ASSERT_EQ(out, - (ByteArray{ - 16, // header - 1, 0, // first item - 2, 0, // second item - 3, 0, // third item - 4, 0 // fourth item - })); - // clang-format on + ASSERT_TRUE(std::ranges::equal(decoded, collection)); + + ByteArray data{ + // clang-format off + 1, 0, // first item + 2, 0, // second item + 3, 0, // third item + 4, 0, // fourth item + // clang-format on + }; + auto match = encodeLen(collection.size()); + match.insert(match.end(), data.begin(), data.end()); + + ASSERT_TRUE(std::ranges::equal(out, match)); } struct TestStruct : std::vector {}; @@ -107,7 +143,7 @@ struct TestStruct : std::vector {}; * @when encodeCollection is applied * @then expected result is obtained */ -TEST(Scale, encodeDerivedCollectionUint16) { +TEST(CollectionTest, encodeDerivedCollectionUint16) { TestStruct collection; collection.push_back(1); collection.push_back(2); @@ -121,19 +157,20 @@ TEST(Scale, encodeDerivedCollectionUint16) { auto stream = ScaleDecoderStream(out); TestStruct decoded; stream >> decoded; - ASSERT_TRUE(std::equal( - decoded.begin(), decoded.end(), collection.begin(), collection.end())); - - // clang-format off - ASSERT_EQ(out, - (ByteArray{ - 16, // header - 1, 0, // first item - 2, 0, // second item - 3, 0, // third item - 4, 0 // fourth item - })); - // clang-format on + ASSERT_TRUE(std::ranges::equal(decoded, collection)); + + ByteArray data{ + // clang-format off + 1, 0, // first item + 2, 0, // second item + 3, 0, // third item + 4, 0, // fourth item + // clang-format on + }; + auto match = encodeLen(collection.size()); + match.insert(match.end(), data.begin(), data.end()); + + ASSERT_TRUE(std::ranges::equal(out, match)); } /** @@ -141,7 +178,7 @@ TEST(Scale, encodeDerivedCollectionUint16) { * @when encodeCollection is applied * @then expected result is obtained */ -TEST(Scale, encodeDequeUint16) { +TEST(CollectionTest, encodeDequeUint16) { std::deque collection = {1, 2, 3, 4}; ScaleEncoderStream s; ASSERT_NO_THROW((s << collection)); @@ -150,19 +187,20 @@ TEST(Scale, encodeDequeUint16) { auto stream = ScaleDecoderStream(out); std::deque decoded; stream >> decoded; - ASSERT_TRUE(std::equal( - decoded.begin(), decoded.end(), collection.begin(), collection.end())); - - // clang-format off - ASSERT_EQ(out, - (ByteArray{ - 16, // header - 1, 0, // first item - 2, 0, // second item - 3, 0, // third item - 4, 0 // fourth item - })); - // clang-format on + ASSERT_TRUE(std::ranges::equal(decoded, collection)); + + ByteArray data{ + // clang-format off + 1, 0, // first item + 2, 0, // second item + 3, 0, // third item + 4, 0, // fourth item + // clang-format on + }; + auto match = encodeLen(collection.size()); + match.insert(match.end(), data.begin(), data.end()); + + ASSERT_TRUE(std::ranges::equal(out, match)); } /** @@ -170,22 +208,30 @@ TEST(Scale, encodeDequeUint16) { * @when encodeCollection is applied * @then expected result is obtained */ -TEST(Scale, encodeCollectionUint32) { +TEST(CollectionTest, encodeCollectionUint32) { std::vector collection = { - 50462976, 117835012, 185207048, 252579084}; + 0x33221100, 0x77665544, 0xbbaa9988, 0xffeeddcc}; ScaleEncoderStream s; ASSERT_NO_THROW((s << collection)); auto &&out = s.to_vector(); - // clang-format off - ASSERT_EQ(out, - (ByteArray{ - 16, // header - 0, 1, 2, 3, // first item - 4, 5, 6, 7, // second item - 8, 9, 0xA, 0xB, // third item - 0xC, 0xD, 0xE, 0xF // fourth item - })); - // clang-format on + + auto stream = ScaleDecoderStream(out); + std::deque decoded; + stream >> decoded; + ASSERT_TRUE(std::ranges::equal(decoded, collection)); + + ByteArray data{ + // clang-format off + 0x00, 0x11, 0x22, 0x33, // first item + 0x44, 0x55, 0x66, 0x77, // second item + 0x88, 0x99, 0xaa, 0xbb, // third item + 0xcc, 0xdd, 0xee, 0xff, // fourth item + // clang-format on + }; + auto match = encodeLen(collection.size()); + match.insert(match.end(), data.begin(), data.end()); + + ASSERT_TRUE(std::ranges::equal(out, match)); } /** @@ -193,22 +239,27 @@ TEST(Scale, encodeCollectionUint32) { * @when encodeCollection is applied * @then expected result is obtained */ -TEST(Scale, encodeCollectionUint64) { - std::vector collection = {506097522914230528ull, - 1084818905618843912ull}; +TEST(CollectionTest, encodeCollectionUint64) { + std::vector collection = {0x7766554433221100, 0xffeeddccbbaa9988}; ScaleEncoderStream s; ASSERT_NO_THROW((s << collection)); auto &&out = s.to_vector(); - // clang-format off - ASSERT_EQ(out, - (ByteArray{ - 8, // header - 0, 1, 2, 3, // first item - 4, 5, 6, 7, // second item - 8, 9, 0xA, 0xB, // third item - 0xC, 0xD, 0xE, 0xF // fourth item - })); - // clang-format on + + auto stream = ScaleDecoderStream(out); + std::deque decoded; + stream >> decoded; + ASSERT_TRUE(std::ranges::equal(decoded, collection)); + + ByteArray data{ + // clang-format off + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, // first item + 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, // second item + // clang-format on + }; + auto match = encodeLen(collection.size()); + match.insert(match.end(), data.begin(), data.end()); + + ASSERT_TRUE(std::ranges::equal(out, match)); } /** @@ -218,7 +269,7 @@ TEST(Scale, encodeCollectionUint64) { * @then obtain byte array of length 32772 bytes * where each second byte == 0 and collection[(i-4)/2] == (i/2) % 256 */ -TEST(Scale, encodeLongCollectionUint16) { +TEST(CollectionTest, encodeLongCollectionUint16) { std::vector collection; auto length = 16384; collection.reserve(length); @@ -229,19 +280,15 @@ TEST(Scale, encodeLongCollectionUint16) { ScaleEncoderStream s; ASSERT_NO_THROW((s << collection)); auto &&out = s.to_vector(); - ASSERT_EQ(out.size(), (length * 2 + 4)); - // header takes 4 byte, - // first 4 bytes represent le-encoded value 2^16 + 2 - // which is compact-encoded value 2^14 = 16384 auto stream = ScaleDecoderStream(out); CompactInteger res{}; ASSERT_NO_THROW(stream >> res); - ASSERT_EQ(res, 16384); + ASSERT_EQ(res, length); // now only 32768 bytes left in stream - ASSERT_EQ(stream.hasMore(32768), true); - ASSERT_EQ(stream.hasMore(32769), false); + ASSERT_EQ(stream.hasMore(length * sizeof(uint16_t)), true); + ASSERT_EQ(stream.hasMore(length * sizeof(uint16_t) + 1), false); for (auto i = 0; i < length; ++i) { uint8_t byte = 0u; @@ -264,11 +311,10 @@ TEST(Scale, encodeLongCollectionUint16) { * where each byte after header == i%256 */ -TEST(Scale, encodeVeryLongCollectionUint8) { +TEST(CollectionTest, encodeVeryLongCollectionUint8) { auto length = 1048576; // 2^20 - std::vector collection; + ByteArray collection; collection.reserve(length); - for (auto i = 0; i < length; ++i) { collection.push_back(i % 256); } @@ -276,12 +322,7 @@ TEST(Scale, encodeVeryLongCollectionUint8) { ScaleEncoderStream s; ASSERT_NO_THROW((s << collection)); auto &&out = s.to_vector(); - ASSERT_EQ(out.size(), (length + 4)); - // header takes 4 bytes, - // first byte == (4-4) + 3 = 3, - // which means that number of items requires 4 bytes - // 3 next bytes are 0, and the last 4-th == 2^6 == 64 - // which is compact-encoded value 2^14 = 16384 + auto stream = ScaleDecoderStream(out); CompactInteger bi{}; ASSERT_NO_THROW(stream >> bi); @@ -300,59 +341,13 @@ TEST(Scale, encodeVeryLongCollectionUint8) { ASSERT_EQ(stream.hasMore(1), false); } -// following test takes too much time, don't run it -/** - * @given very long collection of items of type uint8_t containing - * 2^30 == 1073741824 items this number takes ~ 1 Gb of data where - * collection[i] == i % 256 - * @when encodeCollection is applied - * @then obtain byte array of length 1073741824 + 5 bytes (header) bytes - * where first bytes represent header, other are data itself - * where each byte after header == i%256 - */ -TEST(Scale, DISABLED_encodeVeryLongCollectionUint8) { - auto length = 1073741824; // 2^20 - std::vector collection; - - collection.reserve(length); - for (auto i = 0; i < length; ++i) { - collection.push_back(i % 256); - } - - ScaleEncoderStream s; - ASSERT_NO_THROW((s << collection)); - auto &&out = s.to_vector(); - ASSERT_EQ(out.size(), (length + 4)); - // header takes 4 bytes, - // first byte == (4-4) + 3 = 3, which means that number of items - // requires 4 bytes - // 3 next bytes are 0, and the last 4-th == 2^6 == 64 - // which is compact-encoded value 2^14 = 16384 - auto stream = ScaleDecoderStream(out); - CompactInteger bi{}; - ASSERT_NO_THROW(stream >> bi); - ASSERT_EQ(bi, length); - - // now only 1048576 bytes left in stream - ASSERT_EQ(stream.hasMore(length), true); - ASSERT_EQ(stream.hasMore(length + 1), false); - - for (auto i = 0; i < length; ++i) { - uint8_t byte = 0u; - ASSERT_NO_THROW(stream >> byte); - ASSERT_EQ(byte, i % 256); - } - - ASSERT_EQ(stream.hasMore(1), false); -} - /** * @given map of * @when encodeCollection is applied * @then expected result is obtained: header is 2 byte, items are pairs of 4 * byte elements each */ -TEST(Scale, encodeMapTest) { +TEST(CollectionTest, encodeMapTest) { std::map collection = {{1, 5}, {2, 6}, {3, 7}, {4, 8}}; ScaleEncoderStream s; ASSERT_NO_THROW((s << collection)); @@ -389,7 +384,7 @@ using SizeLimitedVector = * @when decode it to collection limited by size 4, 3 and 2 max size * @then if max_size is enough, it is done successful, and error otherwise */ -TEST(Scale, decodeSizeLimitedCollection) { +TEST(CollectionTest, decodeSizeLimitedCollection) { std::vector collection{1, 2, 3}; ScaleEncoderStream s; @@ -428,7 +423,7 @@ struct ExplicitlyDefinedAsDynamic : public std::vector { using Collection::Collection; }; -TEST(Scale, encodeExplicitlyDefinedAsDynamic) { +TEST(CollectionTest, encodeExplicitlyDefinedAsDynamic) { using TestCollection = ExplicitlyDefinedAsDynamic; const TestCollection collection{1, 2, 3, 4, 5}; @@ -447,7 +442,7 @@ struct ImplicitlyDefinedAsStatic : public std::array { using Collection = std::array; }; -TEST(Scale, encodeImplicitlyDefinedAsStatic) { +TEST(CollectionTest, encodeImplicitlyDefinedAsStatic) { using TestCollection = ImplicitlyDefinedAsStatic; const TestCollection collection{1, 2, 3, 4, 5}; @@ -467,7 +462,7 @@ struct ImplicitlyDefinedAsDynamic : public std::vector { using Collection::Collection; }; -TEST(Scale, encodeImplicitlyDefinedAsDynamic) { +TEST(CollectionTest, encodeImplicitlyDefinedAsDynamic) { using TestCollection = ImplicitlyDefinedAsDynamic; const TestCollection collection{1, 2, 3, 4, 5}; @@ -487,7 +482,7 @@ struct StaticSpan : public std::span { using Collection::Collection; }; -TEST(Scale, encodeStaticSpan) { +TEST(CollectionTest, encodeStaticSpan) { using TestCollection = StaticSpan; std::array original_data{1, 2, 3, 4, 5}; diff --git a/test/scale_compact_test.cpp b/test/scale_compact_test.cpp index c7a578a..abed279 100644 --- a/test/scale_compact_test.cpp +++ b/test/scale_compact_test.cpp @@ -36,8 +36,8 @@ class CompactTest */ TEST_P(CompactTest, EncodeSuccess) { const auto &[value, match] = GetParam(); - ASSERT_NO_THROW(s << value); - ASSERT_EQ(s.to_vector(), match); + ASSERT_NO_THROW(s << value) << "Exception while encoding"; + ASSERT_EQ(s.to_vector(), match) << "Encoding fail"; } /** @@ -49,10 +49,49 @@ TEST_P(CompactTest, DecodeSuccess) { const auto &[value_match, bytes] = GetParam(); ScaleDecoderStream s(bytes); CompactInteger v{}; - ASSERT_NO_THROW(s >> v); - ASSERT_EQ(v, value_match); + ASSERT_NO_THROW(s >> v) << "Exception while decoding"; + ASSERT_EQ(v, value_match) << "Decoding fail"; } +#ifdef JAM_COMPATIBILITY_ENABLED + +#define BIGGEST_INT_FOR_COMPACT_REPRESENTATION \ + ((CompactInteger(1) << (sizeof(size_t) * CHAR_WIDTH)) - 1) +INSTANTIATE_TEST_SUITE_P( + CompactTestCases, + CompactTest, + ::testing::Values( + // clang-format off + + // Lowest values for each number of bytes + /* 0 */ CompactTest::pair( 0b00000000, {0b00000000}), + /* 1 */ CompactTest::pair( 0b10000000, {0b10000000, 0b10000000}), + /* 2 */ CompactTest::pair( 0b01000000'00000000, {0b11000000, 0b00000000, 0b01000000}), + /* 3 */ CompactTest::pair( 0b00100000'00000000'00000000, {0b11100000, 0b00000000, 0b00000000, 0b00100000}), + /* 4 */ CompactTest::pair( 0b00010000'00000000'00000000'00000000, {0b11110000, 0b00000000, 0b00000000, 0b00000000, 0b00010000}), + /* 5 */ CompactTest::pair( 0b00001000'00000000'00000000'00000000'00000000, {0b11111000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00001000}), + /* 6 */ CompactTest::pair( 0b0000100'00000000'00000000'00000000'00000000'00000000, {0b11111100, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000100}), + /* 7 */ CompactTest::pair( 0b0000010'00000000'00000000'00000000'00000000'00000000'00000000, {0b11111110, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000010}), + /* 8 */ CompactTest::pair(0b00000001'00000000'00000000'00000000'00000000'00000000'00000000'00000000, {0b11111111, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000001}), + + // Biggest values for each number of bytes + /* 9 */ CompactTest::pair( 0b01111111, {0b01111111}), + /* 10 */ CompactTest::pair( 0b00111111'11111111, {0b10111111, 0b11111111}), + /* 11 */ CompactTest::pair( 0b00011111'11111111'11111111, {0b11011111, 0b11111111, 0b11111111}), + /* 12 */ CompactTest::pair( 0b00001111'11111111'11111111'11111111, {0b11101111, 0b11111111, 0b11111111, 0b11111111}), + /* 13 */ CompactTest::pair( 0b00000111'11111111'11111111'11111111'11111111, {0b11110111, 0b11111111, 0b11111111, 0b11111111, 0b11111111}), + /* 14 */ CompactTest::pair( 0b00000011'11111111'11111111'11111111'11111111'11111111, {0b11111011, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111}), + /* 15 */ CompactTest::pair( 0b00000001'11111111'11111111'11111111'11111111'11111111'11111111, {0b11111101, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111}), + /* 16 */ CompactTest::pair(0b00000000'11111111'11111111'11111111'11111111'11111111'11111111'11111111, {0b11111110, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111}), + /* 17 */ CompactTest::pair(0b11111111'11111111'11111111'11111111'11111111'11111111'11111111'11111111, {0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111}) + // clang-format on + )); + +#else + +#define BIGGEST_INT_FOR_COMPACT_REPRESENTATION \ + (CompactInteger(1) << (67 * CHAR_WIDTH)) - 1 + INSTANTIATE_TEST_SUITE_P( CompactTestCases, CompactTest, @@ -100,13 +139,11 @@ INSTANTIATE_TEST_SUITE_P( 3}), // min multibyte integer CompactTest::pair(1073741824, {3, 0, 0, 0, 64}), - // max multibyte integer - CompactTest::pair( - CompactInteger( - "224945689727159819140526925384299092943484855915095831" - "655037778630591879033574393515952034305194542857496045" - "531676044756160413302774714984450425759043258192756735"), - std::vector(68, 0xFF)))); + // max multibyte integer: 2^536 - 1 + CompactTest::pair(BIGGEST_INT_FOR_COMPACT_REPRESENTATION, + std::vector(68, 0xFF)))); + +#endif /** * Negative tests @@ -131,16 +168,13 @@ TEST(ScaleCompactTest, EncodeNegativeIntegerFails) { * @then obtain kValueIsTooBig error */ TEST(ScaleCompactTest, EncodeOutOfRangeBigIntegerFails) { - // try to encode out of range big integer value MAX_BIGINT + 1 == 2^536 + // try to encode out of range big integer value MAX_BIGINT + 1 // too big value, even for big integer case // we are going to have kValueIsTooBig error - CompactInteger v( - "224945689727159819140526925384299092943484855915095831" - "655037778630591879033574393515952034305194542857496045" - "531676044756160413302774714984450425759043258192756736"); // 2^536 + CompactInteger v = BIGGEST_INT_FOR_COMPACT_REPRESENTATION + 1; ScaleEncoderStream out; - ASSERT_ANY_THROW((out << v)); // value is too big, it is not encoded + ASSERT_ANY_THROW((out << v)); // value is too big, it isn't encoded ASSERT_EQ(out.to_vector().size(), 0); // nothing was written to buffer } @@ -164,6 +198,35 @@ TEST_P(RedundantCompactTest, DecodeError) { EXPECT_EC(scale::decode(GetParam()), scale::DecodeError::REDUNDANT_COMPACT_ENCODING); } + +#ifdef JAM_COMPATIBILITY_ENABLED + +INSTANTIATE_TEST_SUITE_P( + RedundantCompactTestCases, + RedundantCompactTest, + ::testing::Values( + // clang-format off + /* 1 */ ByteArray{0b10000000, 0b00000000}, + /* 2 */ ByteArray{0b10000000, 0b00111111}, + /* 3 */ ByteArray{0b11000000, 0b00000000, 0b00000000}, + /* 4 */ ByteArray{0b11000000, 0b11111111, 0b00011111}, + /* 5 */ ByteArray{0b11100000, 0b00000000, 0b00000000, 0b00000000}, + /* 6 */ ByteArray{0b11100000, 0b11111111, 0b11111111, 0b00001111}, + /* 7 */ ByteArray{0b11110000, 0b00000000, 0b00000000, 0b00000000, 0b00000000}, + /* 8 */ ByteArray{0b11110000, 0b11111111, 0b11111111, 0b11111111, 0b00000111}, + /* 9 */ ByteArray{0b11111000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000}, + /* 10 */ ByteArray{0b11111000, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b00000011}, + /* 11 */ ByteArray{0b11111100, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000}, + /* 12 */ ByteArray{0b11111100, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b00000001}, + /* 13 */ ByteArray{0b11111110, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000}, + /* 14 */ ByteArray{0b11111110, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b00000000}, + /* 15 */ ByteArray{0b11111111, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00000000}, + /* 16 */ ByteArray{0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b11111111, 0b00000000} + // // clang-format on + )); + +#else + INSTANTIATE_TEST_SUITE_P( RedundantCompactTestCases, RedundantCompactTest, @@ -171,3 +234,6 @@ INSTANTIATE_TEST_SUITE_P( ByteArray{0b000000'10, 0b10000000, 0, 0}, ByteArray{0b000000'11, 0, 0, 0, 0b00'100000}, ByteArray{0b000001'11, 0, 0, 0, 0b01'000000, 0})); + +#endif + diff --git a/test/scale_strings_test.cpp b/test/scale_strings_test.cpp index 03e0253..d11d043 100644 --- a/test/scale_strings_test.cpp +++ b/test/scale_strings_test.cpp @@ -16,11 +16,16 @@ using scale::ScaleEncoderStream; * @when specified string is encoded by ScaleEncoderStream * @then encoded value meets expectations */ -TEST(Scale, RawStringEncodeSuccess) { - auto *v = "asdadad"; +TEST(StringTest, RawStringEncodeSuccess) { + auto *v = "abcdef"; ScaleEncoderStream s{}; ASSERT_NO_THROW((s << v)); - ASSERT_EQ(s.to_vector(), (ByteArray{28, 'a', 's', 'd', 'a', 'd', 'a', 'd'})); + auto buff = s.to_vector(); + + auto actual = std::span(buff).last(buff.size() - 1); + auto expected = ByteArray{'a', 'b', 'c', 'd', 'e', 'f'}; + + ASSERT_TRUE(std::ranges::equal(actual, expected)); } /** @@ -28,11 +33,16 @@ TEST(Scale, RawStringEncodeSuccess) { * @when specified string is encoded by ScaleEncoderStream * @then encoded value meets expectations */ -TEST(Scale, StdStringEncodeSuccess) { - std::string v = "asdadad"; +TEST(StringTest, StdStringEncodeSuccess) { + std::string v = "abcdef"; ScaleEncoderStream s; ASSERT_NO_THROW((s << v)); - ASSERT_EQ(s.to_vector(), (ByteArray{28, 'a', 's', 'd', 'a', 'd', 'a', 'd'})); + auto buff = s.to_vector(); + + auto actual = std::span(buff).last(buff.size() - 1); + auto expected = ByteArray{'a', 'b', 'c', 'd', 'e', 'f'}; + + ASSERT_TRUE(std::ranges::equal(actual, expected)); } /** @@ -40,10 +50,15 @@ TEST(Scale, StdStringEncodeSuccess) { * @when string is decoded using ScaleDecoderStream * @then decoded string matches expectations */ -TEST(Scale, StringDecodeSuccess) { - auto bytes = ByteArray{28, 'a', 's', 'd', 'a', 'd', 'a', 'd'}; - ScaleDecoderStream s(bytes); - std::string v; - ASSERT_NO_THROW(s >> v); - ASSERT_EQ(v, "asdadad"); +TEST(StringTest, StringDecodeSuccess) { + std::string i = "abcdef"; + ScaleEncoderStream es; + ASSERT_NO_THROW((es << i)); + auto encoded = es.to_vector(); + + ScaleDecoderStream ds(encoded); + std::string o; + ds >> o; + + ASSERT_TRUE(std::ranges::equal(i, o)); } diff --git a/test/scale_tune_test.cpp b/test/scale_tune_test.cpp new file mode 100644 index 0000000..6865b7b --- /dev/null +++ b/test/scale_tune_test.cpp @@ -0,0 +1,81 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef CUSTOM_CONFIG_ENABLED +#error \ + "This file should not be compiled, because custom config support is not enabed" +#endif + +#include + +#include + +using scale::ByteArray; +using scale::CompactInteger; +using scale::decode; +using scale::encode; +using scale::ScaleDecoderStream; +using scale::ScaleEncoderStream; + +struct CustomConfig { + uint8_t multi; +}; + +struct Object { + std::vector buff; + + bool operator==(const Object &) const = default; + + friend ScaleEncoderStream &operator<<(ScaleEncoderStream &s, + const Object &x) { + auto config = s.getConfig(); + + s << CompactInteger{x.buff.size()}; + for (uint8_t i : x.buff) { + uint8_t x = i * config.multi; + s << x; + } + return s; + } + + friend ScaleDecoderStream &operator>>(ScaleDecoderStream &s, Object &x) { + auto config = s.getConfig(); + CompactInteger size; + s >> size; + x.buff.resize(size.convert_to()); + for (uint8_t &i : x.buff) { + uint8_t x; + s >> x; + i = x / config.multi; + } + return s; + } +}; + +TEST(CustomConfig, SunnyDayScenario) { + CustomConfig two{2}; + CustomConfig three{3}; + + // Encoding + Object object{.buff = {3, 6, 9}}; + ScaleEncoderStream encoder(two); + encoder << object; + auto encoded = encoder.to_vector(); + + // Check encoding + ScaleDecoderStream s(encoded); + std::vector buff1; + s >> buff1; + std::vector buff2 = {6, 12, 18}; + ASSERT_EQ(buff1, buff2) << "Incorrect encoded"; + + // Decode and check it + ScaleDecoderStream decoder(encoded, three); + Object object1; + decoder >> object1; + Object object2{.buff = {2, 4, 6}}; + ASSERT_EQ(object1, object2) << "Incorrect decoded"; +} From 46f1c355c3a9f7d29a1302a0e1766bf038359f6b Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Tue, 3 Dec 2024 21:13:14 +0800 Subject: [PATCH 02/11] fix: mac CI Signed-off-by: Dmitriy Khaustov aka xDimon --- test/scale_compact_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/scale_compact_test.cpp b/test/scale_compact_test.cpp index abed279..832cc93 100644 --- a/test/scale_compact_test.cpp +++ b/test/scale_compact_test.cpp @@ -56,7 +56,7 @@ TEST_P(CompactTest, DecodeSuccess) { #ifdef JAM_COMPATIBILITY_ENABLED #define BIGGEST_INT_FOR_COMPACT_REPRESENTATION \ - ((CompactInteger(1) << (sizeof(size_t) * CHAR_WIDTH)) - 1) + ((CompactInteger(1) << (8 * sizeof(size_t))) - 1) INSTANTIATE_TEST_SUITE_P( CompactTestCases, CompactTest, @@ -90,7 +90,7 @@ INSTANTIATE_TEST_SUITE_P( #else #define BIGGEST_INT_FOR_COMPACT_REPRESENTATION \ - (CompactInteger(1) << (67 * CHAR_WIDTH)) - 1 + (CompactInteger(1) << (8 * 67)) - 1 INSTANTIATE_TEST_SUITE_P( CompactTestCases, From 0fcd42fbb71485f7285b44dd860feaec5ce0b096 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Mon, 9 Dec 2024 11:54:17 +0800 Subject: [PATCH 03/11] fix: review issues Signed-off-by: Dmitriy Khaustov aka xDimon --- include/scale/scale_decoder_stream.hpp | 11 +++++++---- include/scale/scale_encoder_stream.hpp | 11 +++++++---- src/scale_decoder_stream.cpp | 3 --- src/scale_encoder_stream.cpp | 3 --- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/include/scale/scale_decoder_stream.hpp b/include/scale/scale_decoder_stream.hpp index 492dd97..05c6890 100644 --- a/include/scale/scale_decoder_stream.hpp +++ b/include/scale/scale_decoder_stream.hpp @@ -17,9 +17,12 @@ #include #include -#include #include +#ifdef JAM_COMPATIBILITY_ENABLED #include +#else +#include +#endif #include #include #include @@ -370,8 +373,8 @@ namespace scale { if (not config_.has_value()) { throw std::runtime_error("Stream created without any custom config"); } - if (config_.type().hash_code() - != typeid(std::reference_wrapper).hash_code()) { + if (config_.type() + != typeid(std::reference_wrapper)) { throw std::runtime_error("Stream created with other custom config"); } return std::any_cast>(config_).get(); @@ -380,7 +383,7 @@ namespace scale { template [[deprecated("Scale has compiled without custom config support")]] // const T & - getConfig_() const = delete; + getConfig() const = delete; #endif private: diff --git a/include/scale/scale_encoder_stream.hpp b/include/scale/scale_encoder_stream.hpp index 3e1c122..b0806c0 100644 --- a/include/scale/scale_encoder_stream.hpp +++ b/include/scale/scale_encoder_stream.hpp @@ -16,9 +16,12 @@ #include #include -#include #include +#ifdef JAM_COMPATIBILITY_ENABLED #include +#else +#include +#endif #include #include #include @@ -259,8 +262,8 @@ namespace scale { if (not config_.has_value()) { throw std::runtime_error("Stream created without any custom config"); } - if (config_.type().hash_code() - != typeid(std::reference_wrapper).hash_code()) { + if (config_.type() + != typeid(std::reference_wrapper)) { throw std::runtime_error("Stream created with other custom config"); } return std::any_cast>(config_).get(); @@ -356,4 +359,4 @@ namespace scale { return s << static_cast>(v); } -} // namespace scale +} // namespace scale \ No newline at end of file diff --git a/src/scale_decoder_stream.cpp b/src/scale_decoder_stream.cpp index 5506ff1..31ee17e 100644 --- a/src/scale_decoder_stream.cpp +++ b/src/scale_decoder_stream.cpp @@ -6,9 +6,6 @@ #include -#include -#include - namespace scale { size_t ScaleDecoderStream::decodeLength() { #ifdef JAM_COMPATIBILITY_ENABLED diff --git a/src/scale_encoder_stream.cpp b/src/scale_encoder_stream.cpp index 4f7be77..1ab2d4c 100644 --- a/src/scale_encoder_stream.cpp +++ b/src/scale_encoder_stream.cpp @@ -6,9 +6,6 @@ #include -#include -#include - namespace scale { ScaleEncoderStream::ScaleEncoderStream() : drop_data_{false}, bytes_written_{0} {} From 34da865fb933f8f2c652b9e02d096e56dd491e70 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Mon, 9 Dec 2024 12:48:07 +0800 Subject: [PATCH 04/11] fix: review issue Signed-off-by: Dmitriy Khaustov aka xDimon --- include/scale/configurable.hpp | 47 ++++++++++++++++++++++++++ include/scale/scale_decoder_stream.hpp | 33 ++++-------------- include/scale/scale_encoder_stream.hpp | 38 ++++----------------- include/scale/tune.hpp | 21 ------------ 4 files changed, 60 insertions(+), 79 deletions(-) create mode 100644 include/scale/configurable.hpp delete mode 100644 include/scale/tune.hpp diff --git a/include/scale/configurable.hpp b/include/scale/configurable.hpp new file mode 100644 index 0000000..28442fa --- /dev/null +++ b/include/scale/configurable.hpp @@ -0,0 +1,47 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#ifdef CUSTOM_CONFIG_ENABLED +#include +#endif + +namespace scale { + + class Configurable { + public: + Configurable() = default; + ~Configurable() = default; + + template + explicit Configurable(const T &config) : config_(std::cref(config)) {} + +#ifdef CUSTOM_CONFIG_ENABLED + template + const T &getConfig() const { + if (not config_.has_value()) { + throw std::runtime_error("Stream created without any custom config"); + } + if (config_.type() != typeid(std::reference_wrapper)) { + throw std::runtime_error( + "Stream created with other custom config type"); + } + return std::any_cast>(config_).get(); + } +#else + template + [[deprecated("Scale has compiled without custom config support")]] // + const T & + getConfig() const = delete; +#endif + +#ifdef CUSTOM_CONFIG_ENABLED + const std::any config_{}; +#endif + }; + +} // namespace scale \ No newline at end of file diff --git a/include/scale/scale_decoder_stream.hpp b/include/scale/scale_decoder_stream.hpp index 05c6890..ebe1c0f 100644 --- a/include/scale/scale_decoder_stream.hpp +++ b/include/scale/scale_decoder_stream.hpp @@ -12,7 +12,9 @@ #include #include #include +#include #include +#include #include @@ -23,12 +25,12 @@ #else #include #endif +#include #include #include -#include namespace scale { - class ScaleDecoderStream { + class ScaleDecoderStream: public Configurable { public: // special tag to differentiate decoding streams from others static constexpr auto is_decoder_stream = true; @@ -39,7 +41,7 @@ namespace scale { template requires(std::is_class_v and not std::is_union_v) ScaleDecoderStream(ConstSpanOfBytes data, const ConfigT &config) - : span_{data}, config_(std::cref(config)) {} + : Configurable(config), span_{data} {} #else template requires(std::is_class_v and not std::is_union_v) @@ -367,25 +369,6 @@ namespace scale { return current_index_; } -#ifdef CUSTOM_CONFIG_ENABLED - template - const T &getConfig() const { - if (not config_.has_value()) { - throw std::runtime_error("Stream created without any custom config"); - } - if (config_.type() - != typeid(std::reference_wrapper)) { - throw std::runtime_error("Stream created with other custom config"); - } - return std::any_cast>(config_).get(); - } -#else - template - [[deprecated("Scale has compiled without custom config support")]] // - const T & - getConfig() const = delete; -#endif - private: bool decodeBool(); /** @@ -420,11 +403,7 @@ namespace scale { ByteSpan span_; -#ifdef CUSTOM_CONFIG_ENABLED - const std::any config_{}; -#endif - SizeType current_index_{0}; }; -} // namespace scale +} // namespace scale \ No newline at end of file diff --git a/include/scale/scale_encoder_stream.hpp b/include/scale/scale_encoder_stream.hpp index b0806c0..7e2b6ff 100644 --- a/include/scale/scale_encoder_stream.hpp +++ b/include/scale/scale_encoder_stream.hpp @@ -6,12 +6,12 @@ #pragma once -#ifdef CUSTOM_CONFIG_ENABLED -#include -#endif #include #include #include +#include +#include +#include #include @@ -22,8 +22,8 @@ #else #include #endif +#include #include -#include #include namespace scale { @@ -31,7 +31,7 @@ namespace scale { /** * @class ScaleEncoderStream designed to scale-encode data to stream */ - class ScaleEncoderStream { + class ScaleEncoderStream : public Configurable { public: // special tag to differentiate encoding streams from others static constexpr auto is_encoder_stream = true; @@ -48,13 +48,12 @@ namespace scale { #ifdef CUSTOM_CONFIG_ENABLED template requires(std::is_class_v and not std::is_union_v) - explicit ScaleEncoderStream(const ConfigT &config) - : config_(std::cref(config)) {} + explicit ScaleEncoderStream(const ConfigT &config) : Configurable(config) {} template requires(std::is_class_v and not std::is_union_v) ScaleEncoderStream(const ConfigT &config, bool drop_data) - : config_(std::cref(config)), drop_data_(drop_data) {} + : Configurable(config), drop_data_(drop_data) {} #else template requires(std::is_class_v and not std::is_union_v) @@ -256,25 +255,6 @@ namespace scale { */ ScaleEncoderStream &operator<<(const CompactInteger &v); -#ifdef CUSTOM_CONFIG_ENABLED - template - const T &getConfig() const { - if (not config_.has_value()) { - throw std::runtime_error("Stream created without any custom config"); - } - if (config_.type() - != typeid(std::reference_wrapper)) { - throw std::runtime_error("Stream created with other custom config"); - } - return std::any_cast>(config_).get(); - } -#else - template - [[deprecated("Scale has compiled without custom config support")]] // - const T & - getConfig_() const = delete; -#endif - protected: template void encodeElementOfTuple(const std::tuple &v) { @@ -333,10 +313,6 @@ namespace scale { private: ScaleEncoderStream &encodeOptionalBool(const std::optional &v); -#ifdef CUSTOM_CONFIG_ENABLED - const std::any config_{}; -#endif - const bool drop_data_ = false; std::deque stream_; size_t bytes_written_ = 0; diff --git a/include/scale/tune.hpp b/include/scale/tune.hpp deleted file mode 100644 index 7473399..0000000 --- a/include/scale/tune.hpp +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Copyright Quadrivium LLC - * All Rights Reserved - * SPDX-License-Identifier: Apache-2.0 - */ - -namespace scale::detail { - - enum Compatibility { - Classic, - Jam, - }; -} - -namespace scale::tune { - inline static constexpr detail::Compatibility classic = - detail::Compatibility::Classic; - inline static constexpr detail::Compatibility jam_compatible = - detail::Compatibility::Jam; - -} // namespace scale::tune From a478d4d5c7349b2d7b69a16fd68c663cda215ee9 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Mon, 9 Dec 2024 15:52:44 +0800 Subject: [PATCH 05/11] fix: review issue Signed-off-by: Dmitriy Khaustov aka xDimon --- include/scale/configurable.hpp | 42 ++++++++++++++++++++------ include/scale/scale_decoder_stream.hpp | 17 ++++------- include/scale/scale_encoder_stream.hpp | 22 +++++--------- test/scale_tune_test.cpp | 31 +++++++++++-------- 4 files changed, 66 insertions(+), 46 deletions(-) diff --git a/include/scale/configurable.hpp b/include/scale/configurable.hpp index 28442fa..1290ac1 100644 --- a/include/scale/configurable.hpp +++ b/include/scale/configurable.hpp @@ -9,28 +9,39 @@ #ifdef CUSTOM_CONFIG_ENABLED #include #endif +#include namespace scale { + template + concept MaybeCofing = std::is_class_v and not std::is_union_v; + class Configurable { public: Configurable() = default; ~Configurable() = default; - template - explicit Configurable(const T &config) : config_(std::cref(config)) {} +#ifdef CUSTOM_CONFIG_ENABLED + template + requires (MaybeCofing and ...) + explicit Configurable(const ConfigTs &...configs) { + (addConfig(configs), ...); + } +#else + template + explicit Configurable(const ConfigTs &config...) {} +#endif #ifdef CUSTOM_CONFIG_ENABLED template + requires MaybeCofing const T &getConfig() const { - if (not config_.has_value()) { - throw std::runtime_error("Stream created without any custom config"); - } - if (config_.type() != typeid(std::reference_wrapper)) { + const auto it = configs_.find(typeid(T).hash_code()); + if (it == configs_.end()) { throw std::runtime_error( - "Stream created with other custom config type"); + "Stream was not configured by such custom config type"); } - return std::any_cast>(config_).get(); + return std::any_cast>(it->second).get(); } #else template @@ -40,7 +51,20 @@ namespace scale { #endif #ifdef CUSTOM_CONFIG_ENABLED - const std::any config_{}; + private: + using typeid_hash = decltype(typeid(void).hash_code()); + + template + void addConfig(const ConfigT &config) { + auto [_, added] = + configs_.emplace(typeid(ConfigT).hash_code(), std::cref(config)); + if (not added) { + throw std::runtime_error( + "Stream can be configured by different custom config types only"); + } + } + + std::unordered_map configs_{}; #endif }; diff --git a/include/scale/scale_decoder_stream.hpp b/include/scale/scale_decoder_stream.hpp index ebe1c0f..05b4969 100644 --- a/include/scale/scale_decoder_stream.hpp +++ b/include/scale/scale_decoder_stream.hpp @@ -6,9 +6,6 @@ #pragma once -#ifdef CUSTOM_CONFIG_ENABLED -#include -#endif #include #include #include @@ -30,7 +27,7 @@ #include namespace scale { - class ScaleDecoderStream: public Configurable { + class ScaleDecoderStream : public Configurable { public: // special tag to differentiate decoding streams from others static constexpr auto is_decoder_stream = true; @@ -38,15 +35,13 @@ namespace scale { explicit ScaleDecoderStream(ConstSpanOfBytes data) : span_{data} {} #ifdef CUSTOM_CONFIG_ENABLED - template - requires(std::is_class_v and not std::is_union_v) - ScaleDecoderStream(ConstSpanOfBytes data, const ConfigT &config) - : Configurable(config), span_{data} {} + explicit ScaleDecoderStream(ConstSpanOfBytes data, + const MaybeCofing auto &...config) + : Configurable(config...), span_{data} {} #else - template - requires(std::is_class_v and not std::is_union_v) [[deprecated("Scale has compiled without custom config support")]] // - ScaleDecoderStream(ConstSpanOfBytes data, const ConfigT &config) = delete; + ScaleDecoderStream(ConstSpanOfBytes data, + const MaybeCofing auto &...config) = delete; #endif template diff --git a/include/scale/scale_encoder_stream.hpp b/include/scale/scale_encoder_stream.hpp index 7e2b6ff..bfb7222 100644 --- a/include/scale/scale_encoder_stream.hpp +++ b/include/scale/scale_encoder_stream.hpp @@ -46,24 +46,18 @@ namespace scale { explicit ScaleEncoderStream(bool drop_data); #ifdef CUSTOM_CONFIG_ENABLED - template - requires(std::is_class_v and not std::is_union_v) - explicit ScaleEncoderStream(const ConfigT &config) : Configurable(config) {} - - template - requires(std::is_class_v and not std::is_union_v) - ScaleEncoderStream(const ConfigT &config, bool drop_data) - : Configurable(config), drop_data_(drop_data) {} + explicit ScaleEncoderStream(const MaybeCofing auto &...config) + : Configurable(config...) {} + + ScaleEncoderStream(bool drop_data, const MaybeCofing auto &...config) + : Configurable(config...), drop_data_(drop_data) {} #else - template - requires(std::is_class_v and not std::is_union_v) [[deprecated("Scale has compiled without custom config support")]] // - ScaleEncoderStream(const ConfigT &config) = delete; + ScaleEncoderStream(const MaybeCofing auto &...config) = delete; - template - requires(std::is_class_v and not std::is_union_v) [[deprecated("Scale has compiled without custom config support")]] // - ScaleEncoderStream(const ConfigT &config, bool drop_data) = delete; + ScaleEncoderStream(bool drop_data, + const MaybeCofing auto &...config) = delete; #endif /** diff --git a/test/scale_tune_test.cpp b/test/scale_tune_test.cpp index 6865b7b..9bb16b4 100644 --- a/test/scale_tune_test.cpp +++ b/test/scale_tune_test.cpp @@ -20,9 +20,12 @@ using scale::encode; using scale::ScaleDecoderStream; using scale::ScaleEncoderStream; -struct CustomConfig { +struct MulConfig { uint8_t multi; }; +struct AddConfig { + uint8_t add; +}; struct Object { std::vector buff; @@ -31,51 +34,55 @@ struct Object { friend ScaleEncoderStream &operator<<(ScaleEncoderStream &s, const Object &x) { - auto config = s.getConfig(); + auto mul = s.getConfig().multi; + auto add = s.getConfig().add; s << CompactInteger{x.buff.size()}; for (uint8_t i : x.buff) { - uint8_t x = i * config.multi; + uint8_t x = i * mul + add; s << x; } return s; } friend ScaleDecoderStream &operator>>(ScaleDecoderStream &s, Object &x) { - auto config = s.getConfig(); + auto mul = s.getConfig().multi; + auto add = s.getConfig().add; CompactInteger size; s >> size; x.buff.resize(size.convert_to()); for (uint8_t &i : x.buff) { uint8_t x; s >> x; - i = x / config.multi; + i = (x - add) / mul; } return s; } }; TEST(CustomConfig, SunnyDayScenario) { - CustomConfig two{2}; - CustomConfig three{3}; + MulConfig mul_two{2}; + MulConfig mul_three{3}; + AddConfig add_six{6}; + AddConfig add_twelve{12}; // Encoding Object object{.buff = {3, 6, 9}}; - ScaleEncoderStream encoder(two); - encoder << object; + ScaleEncoderStream encoder(mul_two, add_twelve); + encoder << object; // X * 2 + 12 auto encoded = encoder.to_vector(); // Check encoding ScaleDecoderStream s(encoded); std::vector buff1; s >> buff1; - std::vector buff2 = {6, 12, 18}; + std::vector buff2 = {18, 24, 30}; ASSERT_EQ(buff1, buff2) << "Incorrect encoded"; // Decode and check it - ScaleDecoderStream decoder(encoded, three); + ScaleDecoderStream decoder(encoded, mul_three, add_six); Object object1; decoder >> object1; - Object object2{.buff = {2, 4, 6}}; + Object object2{.buff = {4, 6, 8}}; ASSERT_EQ(object1, object2) << "Incorrect decoded"; } From 820275581bf41a7fc98abe6e7e299659875b9885 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Mon, 9 Dec 2024 16:08:02 +0800 Subject: [PATCH 06/11] fix: broken build Signed-off-by: Dmitriy Khaustov aka xDimon --- include/scale/configurable.hpp | 3 ++- include/scale/scale_decoder_stream.hpp | 6 +++--- include/scale/scale_encoder_stream.hpp | 13 +++++++------ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/include/scale/configurable.hpp b/include/scale/configurable.hpp index 1290ac1..72a1ae5 100644 --- a/include/scale/configurable.hpp +++ b/include/scale/configurable.hpp @@ -29,7 +29,8 @@ namespace scale { } #else template - explicit Configurable(const ConfigTs &config...) {} + requires (MaybeCofing and ...) + explicit Configurable(const ConfigTs &...configs) {} #endif #ifdef CUSTOM_CONFIG_ENABLED diff --git a/include/scale/scale_decoder_stream.hpp b/include/scale/scale_decoder_stream.hpp index 05b4969..4c50d6a 100644 --- a/include/scale/scale_decoder_stream.hpp +++ b/include/scale/scale_decoder_stream.hpp @@ -36,12 +36,12 @@ namespace scale { #ifdef CUSTOM_CONFIG_ENABLED explicit ScaleDecoderStream(ConstSpanOfBytes data, - const MaybeCofing auto &...config) - : Configurable(config...), span_{data} {} + const MaybeCofing auto &...configs) + : Configurable(configs...), span_{data} {} #else [[deprecated("Scale has compiled without custom config support")]] // ScaleDecoderStream(ConstSpanOfBytes data, - const MaybeCofing auto &...config) = delete; + const MaybeCofing auto &...configs) = delete; #endif template diff --git a/include/scale/scale_encoder_stream.hpp b/include/scale/scale_encoder_stream.hpp index bfb7222..a7eab20 100644 --- a/include/scale/scale_encoder_stream.hpp +++ b/include/scale/scale_encoder_stream.hpp @@ -46,18 +46,19 @@ namespace scale { explicit ScaleEncoderStream(bool drop_data); #ifdef CUSTOM_CONFIG_ENABLED - explicit ScaleEncoderStream(const MaybeCofing auto &...config) - : Configurable(config...) {} + explicit ScaleEncoderStream(const MaybeCofing auto &...configs) + : Configurable(configs...) {} - ScaleEncoderStream(bool drop_data, const MaybeCofing auto &...config) - : Configurable(config...), drop_data_(drop_data) {} + explicit ScaleEncoderStream(bool drop_data, + const MaybeCofing auto &...configs) + : Configurable(configs...), drop_data_(drop_data) {} #else [[deprecated("Scale has compiled without custom config support")]] // - ScaleEncoderStream(const MaybeCofing auto &...config) = delete; + ScaleEncoderStream(const MaybeCofing auto &...configs) = delete; [[deprecated("Scale has compiled without custom config support")]] // ScaleEncoderStream(bool drop_data, - const MaybeCofing auto &...config) = delete; + const MaybeCofing auto &...configs) = delete; #endif /** From 1d6a6ec8c10569815945dc37a4f6ead4cf3f0e38 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Mon, 9 Dec 2024 16:08:35 +0800 Subject: [PATCH 07/11] fix: typo Signed-off-by: Dmitriy Khaustov aka xDimon --- include/scale/configurable.hpp | 6 +++--- include/scale/scale_decoder_stream.hpp | 2 +- include/scale/scale_encoder_stream.hpp | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/scale/configurable.hpp b/include/scale/configurable.hpp index 72a1ae5..0036ce9 100644 --- a/include/scale/configurable.hpp +++ b/include/scale/configurable.hpp @@ -14,7 +14,7 @@ namespace scale { template - concept MaybeCofing = std::is_class_v and not std::is_union_v; + concept MaybeConfig = std::is_class_v and not std::is_union_v; class Configurable { public: @@ -23,7 +23,7 @@ namespace scale { #ifdef CUSTOM_CONFIG_ENABLED template - requires (MaybeCofing and ...) + requires (MaybeConfig and ...) explicit Configurable(const ConfigTs &...configs) { (addConfig(configs), ...); } @@ -35,7 +35,7 @@ namespace scale { #ifdef CUSTOM_CONFIG_ENABLED template - requires MaybeCofing + requires MaybeConfig const T &getConfig() const { const auto it = configs_.find(typeid(T).hash_code()); if (it == configs_.end()) { diff --git a/include/scale/scale_decoder_stream.hpp b/include/scale/scale_decoder_stream.hpp index 4c50d6a..6097523 100644 --- a/include/scale/scale_decoder_stream.hpp +++ b/include/scale/scale_decoder_stream.hpp @@ -36,7 +36,7 @@ namespace scale { #ifdef CUSTOM_CONFIG_ENABLED explicit ScaleDecoderStream(ConstSpanOfBytes data, - const MaybeCofing auto &...configs) + const MaybeConfig auto &...configs) : Configurable(configs...), span_{data} {} #else [[deprecated("Scale has compiled without custom config support")]] // diff --git a/include/scale/scale_encoder_stream.hpp b/include/scale/scale_encoder_stream.hpp index a7eab20..7910f53 100644 --- a/include/scale/scale_encoder_stream.hpp +++ b/include/scale/scale_encoder_stream.hpp @@ -46,11 +46,11 @@ namespace scale { explicit ScaleEncoderStream(bool drop_data); #ifdef CUSTOM_CONFIG_ENABLED - explicit ScaleEncoderStream(const MaybeCofing auto &...configs) + explicit ScaleEncoderStream(const MaybeConfig auto &...configs) : Configurable(configs...) {} explicit ScaleEncoderStream(bool drop_data, - const MaybeCofing auto &...configs) + const MaybeConfig auto &...configs) : Configurable(configs...), drop_data_(drop_data) {} #else [[deprecated("Scale has compiled without custom config support")]] // From 03360e12aaef84657a53cf7c947c99943abfacf2 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Mon, 9 Dec 2024 16:18:21 +0800 Subject: [PATCH 08/11] fix: typo Signed-off-by: Dmitriy Khaustov aka xDimon --- include/scale/configurable.hpp | 3 ++- include/scale/scale_decoder_stream.hpp | 2 +- include/scale/scale_encoder_stream.hpp | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/scale/configurable.hpp b/include/scale/configurable.hpp index 0036ce9..1cdab7d 100644 --- a/include/scale/configurable.hpp +++ b/include/scale/configurable.hpp @@ -10,6 +10,7 @@ #include #endif #include +#include namespace scale { @@ -29,7 +30,7 @@ namespace scale { } #else template - requires (MaybeCofing and ...) + requires (MaybeConfig and ...) explicit Configurable(const ConfigTs &...configs) {} #endif diff --git a/include/scale/scale_decoder_stream.hpp b/include/scale/scale_decoder_stream.hpp index 6097523..a7a3d72 100644 --- a/include/scale/scale_decoder_stream.hpp +++ b/include/scale/scale_decoder_stream.hpp @@ -41,7 +41,7 @@ namespace scale { #else [[deprecated("Scale has compiled without custom config support")]] // ScaleDecoderStream(ConstSpanOfBytes data, - const MaybeCofing auto &...configs) = delete; + const MaybeConfig auto &...configs) = delete; #endif template diff --git a/include/scale/scale_encoder_stream.hpp b/include/scale/scale_encoder_stream.hpp index 7910f53..db7b5c7 100644 --- a/include/scale/scale_encoder_stream.hpp +++ b/include/scale/scale_encoder_stream.hpp @@ -54,11 +54,11 @@ namespace scale { : Configurable(configs...), drop_data_(drop_data) {} #else [[deprecated("Scale has compiled without custom config support")]] // - ScaleEncoderStream(const MaybeCofing auto &...configs) = delete; + explicit ScaleEncoderStream(const MaybeConfig auto &...configs) = delete; [[deprecated("Scale has compiled without custom config support")]] // - ScaleEncoderStream(bool drop_data, - const MaybeCofing auto &...configs) = delete; + explicit ScaleEncoderStream(bool drop_data, + const MaybeConfig auto &...configs) = delete; #endif /** From 11e08f1587af3478ac193c7ee12f319bbc196e55 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Mon, 9 Dec 2024 16:40:22 +0800 Subject: [PATCH 09/11] fix: review issue Signed-off-by: Dmitriy Khaustov aka xDimon --- include/scale/configurable.hpp | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/include/scale/configurable.hpp b/include/scale/configurable.hpp index 1cdab7d..002cd52 100644 --- a/include/scale/configurable.hpp +++ b/include/scale/configurable.hpp @@ -9,8 +9,9 @@ #ifdef CUSTOM_CONFIG_ENABLED #include #endif -#include #include +#include +#include namespace scale { @@ -24,13 +25,13 @@ namespace scale { #ifdef CUSTOM_CONFIG_ENABLED template - requires (MaybeConfig and ...) + requires(MaybeConfig and ...) explicit Configurable(const ConfigTs &...configs) { (addConfig(configs), ...); } #else template - requires (MaybeConfig and ...) + requires(MaybeConfig and ...) explicit Configurable(const ConfigTs &...configs) {} #endif @@ -38,7 +39,7 @@ namespace scale { template requires MaybeConfig const T &getConfig() const { - const auto it = configs_.find(typeid(T).hash_code()); + const auto it = configs_.find(typeid(T)); if (it == configs_.end()) { throw std::runtime_error( "Stream was not configured by such custom config type"); @@ -54,19 +55,16 @@ namespace scale { #ifdef CUSTOM_CONFIG_ENABLED private: - using typeid_hash = decltype(typeid(void).hash_code()); - template void addConfig(const ConfigT &config) { - auto [_, added] = - configs_.emplace(typeid(ConfigT).hash_code(), std::cref(config)); + auto [_, added] = configs_.emplace(typeid(ConfigT), std::cref(config)); if (not added) { throw std::runtime_error( "Stream can be configured by different custom config types only"); } } - std::unordered_map configs_{}; + std::unordered_map configs_{}; #endif }; From 2d47f19b4b2a9c31cfa257c3c9d6b0529723e420 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Mon, 9 Dec 2024 20:00:52 +0800 Subject: [PATCH 10/11] fix: review issue Signed-off-by: Dmitriy Khaustov aka xDimon --- include/scale/scale_decoder_stream.hpp | 2 +- include/scale/scale_encoder_stream.hpp | 2 +- src/compact_len_utils.hpp | 33 -------------------------- 3 files changed, 2 insertions(+), 35 deletions(-) delete mode 100644 src/compact_len_utils.hpp diff --git a/include/scale/scale_decoder_stream.hpp b/include/scale/scale_decoder_stream.hpp index a7a3d72..61010af 100644 --- a/include/scale/scale_decoder_stream.hpp +++ b/include/scale/scale_decoder_stream.hpp @@ -401,4 +401,4 @@ namespace scale { SizeType current_index_{0}; }; -} // namespace scale \ No newline at end of file +} // namespace scale diff --git a/include/scale/scale_encoder_stream.hpp b/include/scale/scale_encoder_stream.hpp index db7b5c7..e76dccc 100644 --- a/include/scale/scale_encoder_stream.hpp +++ b/include/scale/scale_encoder_stream.hpp @@ -330,4 +330,4 @@ namespace scale { return s << static_cast>(v); } -} // namespace scale \ No newline at end of file +} // namespace scale diff --git a/src/compact_len_utils.hpp b/src/compact_len_utils.hpp deleted file mode 100644 index f477e33..0000000 --- a/src/compact_len_utils.hpp +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright Quadrivium LLC - * All Rights Reserved - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include - -namespace scale::compact { -// // calculate number of bytes required -// inline size_t countBytes(CompactInteger v) { -// size_t counter = 0; -// do { -// ++counter; -// } while ((v >>= 8) != 0); -// return counter; -// } -// -// /** -// * Returns the compact encoded length for the given value. -// */ -// template , -// typename = std::enable_if_t::value>> -// uint32_t compactLen(T val) { -// if (val < EncodingCategoryLimits::kMinUint16) return 1; -// if (val < EncodingCategoryLimits::kMinUint32) return 2; -// if (val < EncodingCategoryLimits::kMinBigInteger) return 4; -// return countBytes(val); -// } -} // namespace scale::compact From ce7f144519226b970e75cf688621921b44c2077f Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Mon, 9 Dec 2024 20:06:21 +0800 Subject: [PATCH 11/11] bump