Merge pull request #1830 from FISCO-BCOS/release-2.7.1

sync code from release-2.7.1 into master
FISCO-BCOS · Dec 7, 2020 · 50a7ffb · 50a7ffb
2 parents 0bc4766 + 14b828d
commit 50a7ffb
Showing 26 changed files with 728 additions and 35 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -170,10 +170,11 @@ jobs:
       - run:
           name: Upload binary
           command: |
+            cp tools/build_chain.sh build/bin/
             cd build/bin
-            tar -zcf fisco-bcos.tar.gz fisco-bcos
+            tar -zcf fisco-bcos.tar.gz fisco-bcos build_chain.sh
             strip fisco-bcos
-            tar -zcf fisco-bcos-mini.tar.gz fisco-bcos
+            tar -zcf fisco-bcos-mini.tar.gz fisco-bcos build_chain.sh
       - store_artifacts:
           path: /FISCO-BCOS/build/bin/fisco-bcos.tar.gz
           destination: fisco-bcos.tar.gz

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -24,7 +24,7 @@ set(FISCO_BCOS_CMAKE_DIR "${CMAKE_CURRENT_LIST_DIR}/cmake" CACHE PATH "The path
 list(APPEND CMAKE_MODULE_PATH ${FISCO_BCOS_CMAKE_DIR})
 set(CMAKE_OSX_DEPLOYMENT_TARGET "10.13" CACHE STRING "Minimum OS X deployment version")
 
-project(FISCO-BCOS VERSION "2.7.0")
+project(FISCO-BCOS VERSION "2.7.1")
 # Suffix like "-rc1" e.t.c. to append to versions wherever needed.
 set(VERSION_SUFFIX "")
 

diff --git a/Changelog.md b/Changelog.md
@@ -1,3 +1,31 @@
+### 2.7.1
+(2020-12-07)
+
+**更改**
+
+- 完善`fisco-sync`工具，实现新节点加入群组时从[数据仓库](https://data-doc.readthedocs.io/zh_CN/latest/docs/WeBankBlockchain-Data-Stash/index.html)拉取指定块高快照数据的功能
+
+**修复**
+
+- 修复mysql存储模式下，系统表缺少hash_2_block表信息的问题
+- 修复scalable存储模式下，从数据仓库拉取数据后的格式转换问题
+
+**兼容性**
+
+**向前兼容**，旧版本可以直接替换程序升级
+
+|            | 推荐版本                | 最低版本  | 说明                   |
+| ---------- | ----------------------- | --------- | ---------------------- |
+| 控制台     | 2.7.0                  | 1.0.4     |                        |
+| Java SDK        | 2.7.0                   | 2.6.1     |                        |
+| WebSDK        | 2.6.0                   | 2.0.4     |                        |
+| generator  | 1.7.0                   | 1.1.0     | 搭建新链需要使用该版本 |
+| 浏览器     | 2.0.2                   | 2.0.0-rc2 |                        |
+| Solidity   | 最高支持 solidity 0.6.10 | 0.4.11    |                        |
+| amdb-proxy | 2.3.0                   | 2.0.2     |                        |
+
+更多，请参考[版本与兼容性描述](https://fisco-bcos-documentation.readthedocs.io/zh_CN/latest/docs/change_log/index.html)。
+
 ### 2.7.0
 (2020-11-20)
 

diff --git a/fisco-bcos/main/DataSyncTool.cpp b/fisco-bcos/main/DataSyncTool.cpp
@@ -51,6 +51,7 @@ namespace fs = boost::filesystem;
 
 uint32_t PageCount = 10000;
 uint32_t BigTablePageCount = 50;
+uint32_t MinVerifyBlocks = 0;
 const string SYNCED_BLOCK_NUMBER = "#extra_synce_block_number#";
 const std::vector<std::string> ForceTables = {
     SYS_HASH_2_BLOCK, SYS_BLOCK_2_NONCES, SYS_TX_HASH_2_BLOCK};
@@ -224,14 +225,13 @@ TableData::Ptr getHashToBlockData(SQLStorage::Ptr _reader, int64_t _blockNumber)
     }
 }
 
-void conversionData(
-    const std::string& tableName, const std::string& dbType, TableData::Ptr tableData)
+void conversionData(const std::string& tableName, TableData::Ptr tableData)
 {
     // do the conversion from Hex to Byte in following situations:
     //     table in _sys_hash_2_block_,_sys_block_2_nonces_ and
     //     support version >= 2.2.0 and
     //     storage type in rocksdb,scalable
-    if (HexTables.end() != find(HexTables.begin(), HexTables.end(), tableName) &&
+    /*if (HexTables.end() != find(HexTables.begin(), HexTables.end(), tableName) &&
         g_BCOSConfig.version() >= V2_2_0 && dev::stringCmpIgnoreCase(dbType, "mysql"))
     {
         LOG(INFO) << LOG_BADGE("STORAGE") << LOG_DESC("conversion table data");
@@ -242,7 +242,52 @@ void conversionData(
             auto dataBytes = std::make_shared<bytes>(fromHex(dataStr.c_str()));
             entry->setField("value", dataBytes->data(), dataBytes->size());
         }
+    }*/
+    if (HexTables.end() != find(HexTables.begin(), HexTables.end(), tableName))
+    {
+        LOG(TRACE) << LOG_BADGE("STORAGE") << LOG_DESC("conversion table data") << LOG_KV("table name", tableName) << LOG_KV("new entry count", tableData->newEntries->size()) << LOG_KV("dirty entry count", tableData->dirtyEntries->size());
+        for (size_t i = 0; i < tableData->newEntries->size(); i++)
+        {
+            auto entry = tableData->newEntries->get(i);
+            auto dataStr = entry->getField("value");
+            auto dataBytes = std::make_shared<bytes>(fromHex(dataStr.c_str()));
+            entry->setField("value", dataBytes->data(), dataBytes->size());
+        }
+        for (size_t i = 0; i < tableData->dirtyEntries->size(); i++)
+        {
+            auto entry = tableData->dirtyEntries->get(i);
+            auto dataStr = entry->getField("value");
+            auto dataBytes = std::make_shared<bytes>(fromHex(dataStr.c_str()));
+            entry->setField("value", dataBytes->data(), dataBytes->size());
+        }
+    } else if (tableName == SYS_HASH_2_BLOCKHEADER) {
+        LOG(TRACE) << LOG_BADGE("STORAGE") << LOG_DESC("conversion table data") << LOG_KV("table name", tableName) << LOG_KV("new entry count", tableData->newEntries->size()) << LOG_KV("dirty entry count", tableData->dirtyEntries->size());
+        for (size_t i = 0; i < tableData->newEntries->size(); i++)
+        {
+            auto entry = tableData->newEntries->get(i);
+            auto dataStr = entry->getField("value");
+            auto dataBytes = std::make_shared<bytes>(fromHex(dataStr.c_str()));
+            entry->setField("value", dataBytes->data(), dataBytes->size());
+            auto dataStr2 = entry->getField("sigs");
+            auto dataBytes2 = std::make_shared<bytes>(fromHex(dataStr2.c_str()));
+            entry->setField("sigs", dataBytes2->data(), dataBytes2->size());
+        }
+    } else if (tableName.substr(0, 2) == "c_") {
+        LOG(TRACE) << LOG_BADGE("STORAGE") << LOG_DESC("conversion table data") << LOG_KV("table name", tableName) << LOG_KV("new entry count", tableData->newEntries->size()) << LOG_KV("dirty entry count", tableData->dirtyEntries->size());
+        for (size_t i = 0; i < tableData->newEntries->size(); i++)
+        {
+            auto entry = tableData->newEntries->get(i);
+            auto dataKey = entry->getField("key");
+            if (dataKey == "code" || dataKey == "codeHash") {
+                auto dataStr = entry->getField("value");
+                if (dataStr.size() > 0) {
+                    auto dataBytes = std::make_shared<bytes>(fromHex(dataStr.c_str()));
+                    entry->setField("value", dataBytes->data(), dataBytes->size());
+                }
+            }
+        }
     }
+    LOG(TRACE) << LOG_BADGE("STORAGE") << LOG_DESC("conversion end!");
 }
 
 void syncData(SQLStorage::Ptr _reader, Storage::Ptr _writer, int64_t _blockNumber,
@@ -299,7 +344,7 @@ void syncData(SQLStorage::Ptr _reader, Storage::Ptr _writer, int64_t _blockNumbe
                      << "] " << tableInfo->name << " downloaded items : " << downloaded << flush;
                 break;
             }
-            conversionData(tableInfo->name, _param->mutableStorageParam().type, tableData);
+            conversionData(tableInfo->name, tableData);
             if (ForceTables.end() != find(ForceTables.begin(), ForceTables.end(), tableInfo->name))
             {
                 for (size_t i = 0; i < tableData->newEntries->size(); i++)
@@ -338,7 +383,7 @@ void syncData(SQLStorage::Ptr _reader, Storage::Ptr _writer, int64_t _blockNumbe
         if (!_fullSync)
         {
             auto data = getHashToBlockData(_reader, syncBlock);
-            conversionData(SYS_HASH_2_BLOCK, _param->mutableStorageParam().type, data);
+            conversionData(SYS_HASH_2_BLOCK, data);
             _writer->commit(syncBlock, vector<TableData::Ptr>{data});
             recorder->markStatus(SYS_HASH_2_BLOCK, make_pair(data->newEntries->size(), true));
         }
@@ -357,7 +402,7 @@ void syncData(SQLStorage::Ptr _reader, Storage::Ptr _writer, int64_t _blockNumbe
             auto data = getBlockToNonceData(_reader, syncBlock);
             if (data)
             {
-                conversionData(SYS_BLOCK_2_NONCES, _param->mutableStorageParam().type, data);
+                conversionData(SYS_BLOCK_2_NONCES, data);
                 _writer->commit(syncBlock, vector<TableData::Ptr>{data});
                 recorder->markStatus(SYS_BLOCK_2_NONCES, make_pair(data->newEntries->size(), true));
             }
@@ -388,6 +433,12 @@ void syncData(SQLStorage::Ptr _reader, Storage::Ptr _writer, int64_t _blockNumbe
 void fastSyncGroupData(std::shared_ptr<LedgerParamInterface> _param,
     ChannelRPCServer::Ptr _channelRPCServer, int64_t _rollbackNumber = 1000)
 {
+    if (g_BCOSConfig.version() < V2_6_0)
+    {
+        cout << "error unsupported version < 2.6.0" << endl;
+        exit(0);
+    }
+
     // create SQLStorage
     auto sqlStorage = createSQLStorage(_param, _channelRPCServer, [](std::exception& e) {
         LOG(ERROR) << LOG_BADGE("STORAGE") << LOG_BADGE("MySQL")
@@ -467,7 +518,7 @@ int main(int argc, const char* argv[])
         boost::program_options::value<std::string>()->default_value("./config.ini"),
         "config file path, eg. config.ini")("verify,v",
         boost::program_options::value<int64_t>()->default_value(1000),
-        "verify number of blocks, minimum is 100")("limit,l",
+        "verify number of blocks, default 1000")("limit,l",
         boost::program_options::value<uint32_t>()->default_value(10000), "page counts of table")(
         "sys_limit,s", boost::program_options::value<uint32_t>()->default_value(50),
         "page counts of system table")(
@@ -490,7 +541,7 @@ int main(int argc, const char* argv[])
         exit(0);
     }
     int64_t verifyBlocks = vm["verify"].as<int64_t>();
-    verifyBlocks = verifyBlocks < 100 ? 100 : verifyBlocks;
+    verifyBlocks = verifyBlocks < MinVerifyBlocks ? MinVerifyBlocks : verifyBlocks;
     PageCount = vm["limit"].as<uint32_t>();
     BigTablePageCount = vm["sys_limit"].as<uint32_t>();
     string configPath = vm["config"].as<std::string>();