1.logging files name ends with the timestamp;

2.make logger.count/logger.size dynamic updatable
pingcap · Sep 17, 2021 · e7f4997 · e7f4997
1 parent 5a1bef9
commit e7f4997
Showing 9 changed files with 276 additions and 11 deletions.
diff --git a/contrib/poco b/contrib/poco
diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt
@@ -266,9 +266,9 @@ if (ENABLE_TESTS)
         ${ClickHouse_SOURCE_DIR}/dbms/src/TestUtils/FunctionTestUtils.cpp
         ${ClickHouse_SOURCE_DIR}/dbms/src/AggregateFunctions/AggregateFunctionSum.cpp
     )
-    target_include_directories(gtests_dbms BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
+    target_include_directories(gtests_dbms BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR} ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
     target_compile_definitions(gtests_dbms PUBLIC DBMS_PUBLIC_GTEST)
-    target_link_libraries(gtests_dbms gtest_main dbms clickhouse_functions)
+    target_link_libraries(gtests_dbms gtest_main dbms clickhouse_functions daemon)
     target_compile_options(gtests_dbms PRIVATE -Wno-unknown-pragmas -Wno-deprecated-copy)
     add_check(gtests_dbms)
 endif ()

diff --git a/dbms/src/Common/MutableSplitterChannel.cpp b/dbms/src/Common/MutableSplitterChannel.cpp
@@ -0,0 +1,46 @@
+#include "MutableSplitterChannel.h"
+
+#include <Poco/Ext/LevelFilterChannel.h>
+#include <Poco/FormattingChannel.h>
+#include <fmt/format.h>
+
+#include <iostream>
+
+#include "TiflashLogFileChannel.h"
+namespace DB
+{
+void MutableSplitterChannel::setPropertiesRecursively(Poco::Logger & logger, Poco::Channel & channel, Poco::Util::AbstractConfiguration & config)
+{
+    if (typeid(channel) == typeid(TiflashLogFileChannel))
+    {
+        TiflashLogFileChannel * fileChannel = dynamic_cast<TiflashLogFileChannel *>(&channel);
+        std::string rotation_size = config.getRawString("logger.size", "100M");
+        std::string purge_count = config.getRawString("logger.count", "1");
+        logger.information(fmt::format("set channel rotation:{}, purgecount:{}", rotation_size, purge_count));
+        fileChannel->setProperty(Poco::FileChannel::PROP_ROTATION, rotation_size);
+        fileChannel->setProperty(Poco::FileChannel::PROP_PURGECOUNT, purge_count);
+        return;
+    }
+    if (typeid(channel) == typeid(Poco::LevelFilterChannel))
+    {
+        Poco::LevelFilterChannel * levelFilterChannel = dynamic_cast<Poco::LevelFilterChannel *>(&channel);
+        setPropertiesRecursively(logger, *levelFilterChannel->getChannel(), config);
+        return;
+    }
+    if (typeid(channel) == typeid(Poco::FormattingChannel))
+    {
+        Poco::FormattingChannel * formattingChannel = dynamic_cast<Poco::FormattingChannel *>(&channel);
+        setPropertiesRecursively(logger, *formattingChannel->getChannel(), config);
+        return;
+    }
+    logger.information(fmt::format("invalid channel type:{}", typeid(channel).name()));
+}
+// only support FormattingChannel --> LevelFilterChannel --> TiflashLogFileChannel or LevelFilterChannel --> TiflashLogFileChannel or TiflashLogFileChannel
+void MutableSplitterChannel::changeProperties(Poco::Logger & logger, Poco::Util::AbstractConfiguration & config)
+{
+    for (auto it : _channels)
+    {
+        setPropertiesRecursively(logger, *it, config);
+    }
+}
+} // namespace DB
diff --git a/dbms/src/Common/MutableSplitterChannel.h b/dbms/src/Common/MutableSplitterChannel.h
@@ -0,0 +1,30 @@
+#pragma once
+
+
+#include <Poco/Logger.h>
+#include <Poco/SplitterChannel.h>
+#include <Poco/Util/AbstractConfiguration.h>
+
+#include <functional>
+namespace DB
+{
+class MutableSplitterChannel : public Poco::SplitterChannel
+{
+public:
+    using SplitterChannelValidator = std::function<void(Poco::Channel &, Poco::Util::AbstractConfiguration &)>;
+    void changeProperties(Poco::Logger & logger, Poco::Util::AbstractConfiguration & config);
+    // just for test now
+    void setPropertiesValidator(SplitterChannelValidator validator) { properties_validator = validator; }
+    void validateProperties(Poco::Util::AbstractConfiguration & expect_config)
+    {
+        for (auto it : _channels)
+        {
+            properties_validator(*it, expect_config);
+        }
+    }
+
+protected:
+    void setPropertiesRecursively(Poco::Logger & logger, Poco::Channel & channel, Poco::Util::AbstractConfiguration & config);
+    SplitterChannelValidator properties_validator = nullptr; // just for test now
+};
+} // namespace DB
diff --git a/dbms/src/Common/TiflashArchiveByTimestampsStrategy.h b/dbms/src/Common/TiflashArchiveByTimestampsStrategy.h
@@ -0,0 +1,25 @@
+#pragma once
+#include <Poco/ArchiveStrategy.h>
+namespace DB
+{
+template <class DT>
+class TiflashArchiveByTimestampsStrategy : public Poco::ArchiveByTimestampStrategy<DT>
+{
+public:
+    Poco::LogFile * archive(Poco::LogFile * pFile) override
+    {
+        std::string path = pFile->path();
+        delete pFile;
+        std::string archPath = path;
+        archPath.append(".");
+        Poco::DateTimeFormatter::append(archPath, DT().timestamp(), "%Y-%m-%d-%H:%M:%S.%i");
+
+        if (Poco::ArchiveByTimestampStrategy<DT>::exists(archPath))
+            Poco::ArchiveByTimestampStrategy<DT>::archiveByNumber(archPath);
+        else
+            Poco::ArchiveByTimestampStrategy<DT>::moveFile(path, archPath);
+
+        return new Poco::LogFile(path);
+    }
+};
+} // namespace DB
diff --git a/dbms/src/Common/TiflashLogFileChannel.cpp b/dbms/src/Common/TiflashLogFileChannel.cpp
@@ -0,0 +1,33 @@
+#include "TiflashLogFileChannel.h"
+
+#include <Poco/LocalDateTime.h>
+
+#include "Poco/ArchiveStrategy.h"
+#include "Poco/String.h"
+#include "TiflashArchiveByTimestampsStrategy.h"
+namespace DB
+{
+void TiflashLogFileChannel::setArchive(const std::string & archive)
+{
+    Poco::ArchiveStrategy * pStrategy = 0;
+    if (archive == "number")
+    {
+        pStrategy = new Poco::ArchiveByNumberStrategy;
+    }
+    else if (archive == "timestamp")
+    {
+        if (_times == "utc")
+            pStrategy = new TiflashArchiveByTimestampsStrategy<Poco::DateTime>;
+        else if (_times == "local")
+            pStrategy = new TiflashArchiveByTimestampsStrategy<Poco::LocalDateTime>;
+        else
+            throw Poco::PropertyNotSupportedException("times", _times);
+    }
+    else
+        throw Poco::InvalidArgumentException("archive", archive);
+    delete _pArchiveStrategy;
+    pStrategy->compress(_compress);
+    _pArchiveStrategy = pStrategy;
+    _archive = archive;
+}
+} // namespace DB
diff --git a/dbms/src/Common/TiflashLogFileChannel.h b/dbms/src/Common/TiflashLogFileChannel.h
@@ -0,0 +1,10 @@
+#pragma once
+#include <Poco/FileChannel.h>
+namespace DB
+{
+class TiflashLogFileChannel : public Poco::FileChannel
+{
+protected:
+    void setArchive(const std::string & archive) override;
+};
+} // namespace DB
diff --git a/dbms/src/Server/tests/gtest_storage_config.cpp b/dbms/src/Server/tests/gtest_storage_config.cpp
@@ -10,7 +10,10 @@
 
 #include <Common/Config/ConfigProcessor.h>
 #include <Common/Config/TOMLConfiguration.h>
+#include <Common/MutableSplitterChannel.h>
+#include <Common/TiflashLogFileChannel.h>
 #include <Interpreters/Quota.h>
+#include <Poco/Ext/LevelFilterChannel.h>
 #include <Poco/Logger.h>
 #include <Poco/Util/LayeredConfiguration.h>
 #include <Server/StorageConfigParser.h>
@@ -22,7 +25,7 @@
 #include <Storages/Transaction/RegionPersister.h>
 #include <Storages/Transaction/TiKVRecordFormat.h>
 #include <TestUtils/TiFlashTestBasic.h>
-
+#include <daemon/BaseDaemon.h>
 namespace DB
 {
 namespace tests
@@ -799,5 +802,102 @@ dt_page_gc_low_write_prob = 0.2
     global_ctx.setSettings(origin_settings);
 }
 CATCH
+
+class TestMutableSplitterChannel : public MutableSplitterChannel
+{
+public:
+};
+
+static void verifyChannelConfig(Poco::Channel & channel, Poco::Util::AbstractConfiguration & config)
+{
+    if (typeid(channel) == typeid(TiflashLogFileChannel))
+    {
+        TiflashLogFileChannel * fileChannel = dynamic_cast<TiflashLogFileChannel *>(&channel);
+        ASSERT_EQ(fileChannel->getProperty(Poco::FileChannel::PROP_ROTATION), config.getRawString("logger.size", "100M"));
+        ASSERT_EQ(fileChannel->getProperty(Poco::FileChannel::PROP_PURGECOUNT), config.getRawString("logger.count", "1"));
+        return;
+    }
+    if (typeid(channel) == typeid(Poco::LevelFilterChannel))
+    {
+        Poco::LevelFilterChannel * levelFilterChannel = dynamic_cast<Poco::LevelFilterChannel *>(&channel);
+        verifyChannelConfig(*levelFilterChannel->getChannel(), config);
+        return;
+    }
+    if (typeid(channel) == typeid(Poco::FormattingChannel))
+    {
+        Poco::FormattingChannel * formattingChannel = dynamic_cast<Poco::FormattingChannel *>(&channel);
+        verifyChannelConfig(*formattingChannel->getChannel(), config);
+    }
+}
+
+TEST_F(UsersConfigParser_test, ReloadLoggerConfig)
+try
+{
+    Strings tests = {
+        R"(
+[profiles]
+[profiles.default]
+max_rows_in_set = 455
+dt_page_gc_low_write_prob = 0.2
+[logger]
+count = 20
+errorlog = "./tmp/log/tiflash_error.log"
+level = "debug"
+log = "./tmp/log/tiflash.log"
+size = "1M"
+        )",
+        R"(
+[application]
+runAsDaemon = false
+[profiles]
+[profiles.default]
+max_rows_in_set = 455
+dt_page_gc_low_write_prob = 0.2
+[logger]
+count = 10
+errorlog = "./tmp/log/tiflash_error.log"
+level = "debug"
+log = "./tmp/log/tiflash.log"
+size = "1K"
+        )",
+        R"(
+[profiles]
+[profiles.default]
+max_rows_in_set = 455
+dt_page_gc_low_write_prob = 0.2
+[logger]
+count = 1
+errorlog = "./tmp/log/tiflash_error.log"
+level = "debug"
+log = "./tmp/log/tiflash.log"
+size = "1"
+        )",
+    };
+    BaseDaemon app;
+
+    auto verifyLoggersConfig = [](size_t logger_num, Poco::Util::AbstractConfiguration & config) {
+        for (size_t j = 0; j < logger_num; j++)
+        {
+            Poco::Logger & cur_logger = Poco::Logger::get(fmt::format("ReloadLoggerConfig_test{}", j));
+            ASSERT_NE(cur_logger.getChannel(), nullptr);
+            Poco::Channel & cur_logger_channel = *cur_logger.getChannel();
+            ASSERT_EQ(typeid(cur_logger_channel), typeid(DB::MutableSplitterChannel));
+            DB::MutableSplitterChannel * splitter_channel = dynamic_cast<DB::MutableSplitterChannel *>(&cur_logger_channel);
+            splitter_channel->setPropertiesValidator(verifyChannelConfig);
+            splitter_channel->validateProperties(config);
+        }
+    };
+
+    for (size_t i = 0; i < tests.size(); ++i)
+    {
+        const auto & test_case = tests[i];
+        auto config = loadConfigFromString(test_case);
+        app.buildLoggers(*config);
+        Poco::Logger::get(fmt::format("ReloadLoggerConfig_test{}", i));
+        LOG_INFO(log, "parsing [index=" << i << "] [content=" << test_case << "]");
+        verifyLoggersConfig(i + 1, *config);
+    }
+}
+CATCH
 } // namespace tests
 } // namespace DB
diff --git a/libs/libdaemon/src/BaseDaemon.cpp b/libs/libdaemon/src/BaseDaemon.cpp
@@ -21,7 +21,9 @@
 #endif
 #include <Common/ClickHouseRevision.h>
 #include <Common/Exception.h>
+#include <Common/MutableSplitterChannel.h>
 #include <Common/TiFlashBuildInfo.h>
+#include <Common/TiflashLogFileChannel.h>
 #include <Common/UnifiedLogPatternFormatter.h>
 #include <Common/getMultipleKeysFromConfig.h>
 #include <Common/setThreadName.h>
@@ -54,6 +56,7 @@
 #include <common/ErrorHandlers.h>
 #include <common/logger_useful.h>
 #include <daemon/OwnPatternFormatter.h>
+#include <fmt/format.h>
 #include <sys/resource.h>
 #include <sys/time.h>
 #include <ucontext.h>
@@ -725,7 +728,6 @@ void BaseDaemon::wakeup()
     wakeup_event.set();
 }
 
-
 void BaseDaemon::buildLoggers(Poco::Util::AbstractConfiguration & config)
 {
     auto current_logger = config.getString("logger");
@@ -736,9 +738,10 @@ void BaseDaemon::buildLoggers(Poco::Util::AbstractConfiguration & config)
     bool is_daemon = config.getBool("application.runAsDaemon", false);
 
     // Split log and error log.
-    Poco::AutoPtr<SplitterChannel> split = new SplitterChannel;
+    Poco::AutoPtr<DB::MutableSplitterChannel> split = new DB::MutableSplitterChannel;
 
     auto log_level = normalize(config.getString("logger.level", "debug"));
+    auto rotation = config.getRawString("logger.size", "100M");
     const auto log_path = config.getString("logger.log", "");
     if (!log_path.empty())
     {
@@ -749,10 +752,11 @@ void BaseDaemon::buildLoggers(Poco::Util::AbstractConfiguration & config)
         Poco::AutoPtr<DB::UnifiedLogPatternFormatter> pf = new DB::UnifiedLogPatternFormatter();
         pf->setProperty("times", "local");
         Poco::AutoPtr<FormattingChannel> log = new FormattingChannel(pf);
-        log_file = new FileChannel;
+        log_file = new DB::TiflashLogFileChannel;
         log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(log_path).absolute().toString());
         log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M"));
-        log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number");
+        log_file->setProperty(Poco::FileChannel::PROP_TIMES, "local");
+        log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "timestamp");
         log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true"));
         log_file->setProperty(Poco::FileChannel::PROP_PURGECOUNT, config.getRawString("logger.count", "1"));
         log_file->setProperty(Poco::FileChannel::PROP_FLUSH, config.getRawString("logger.flush", "true"));
@@ -772,10 +776,11 @@ void BaseDaemon::buildLoggers(Poco::Util::AbstractConfiguration & config)
         Poco::AutoPtr<DB::UnifiedLogPatternFormatter> pf = new DB::UnifiedLogPatternFormatter();
         pf->setProperty("times", "local");
         Poco::AutoPtr<FormattingChannel> errorlog = new FormattingChannel(pf);
-        error_log_file = new FileChannel;
+        error_log_file = new DB::TiflashLogFileChannel;
         error_log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(errorlog_path).absolute().toString());
         error_log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M"));
-        error_log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number");
+        error_log_file->setProperty(Poco::FileChannel::PROP_TIMES, "local");
+        error_log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "timestamp");
         error_log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true"));
         error_log_file->setProperty(Poco::FileChannel::PROP_PURGECOUNT, config.getRawString("logger.count", "1"));
         error_log_file->setProperty(Poco::FileChannel::PROP_FLUSH, config.getRawString("logger.flush", "true"));
@@ -822,7 +827,23 @@ void BaseDaemon::buildLoggers(Poco::Util::AbstractConfiguration & config)
     std::vector<std::string> names;
     Logger::root().names(names);
     for (const auto & name : names)
-        Logger::root().get(name).setLevel(log_level);
+    {
+        Logger & cur_logger = Logger::root().get(name);
+        cur_logger.setLevel(log_level);
+        if (!cur_logger.getChannel())
+        {
+            continue;
+        }
+        Poco::Channel & cur_logger_channel = *cur_logger.getChannel();
+        logger().information(fmt::format("logger name:{}, type:{}", name, typeid(cur_logger_channel).name()));
+        // only loggers created after buildLoggers() need to change properties, types of channel in them must be MutableSplitterChannel
+        // typeid should be applied to a polymorphic class type but not a pointer, see https://stackoverflow.com/questions/17010884/typeid-for-polymorphic-pointers
+        if (typeid(cur_logger_channel) == typeid(DB::MutableSplitterChannel))
+        {
+            DB::MutableSplitterChannel * splitter_channel = dynamic_cast<DB::MutableSplitterChannel *>(&cur_logger_channel);
+            splitter_channel->changeProperties(logger(), config);
+        }
+    }
 
     // Attach to the root logger.
     Logger::root().setLevel(log_level);
+2 −2		Foundation/include/Poco/ArchiveStrategy.h
+1 −3		Foundation/include/Poco/FileChannel.h
+0 −2		Foundation/include/Poco/SplitterChannel.h