Apply time benchmarking (#4402)

# Description Resolves #4391.  # Checklist - [ ] Reviewed the [contributing](https://github.com/stellar/stellar-core/blob/master/CONTRIBUTING.md#submitting-changes) document - [ ] Rebased on top of master (no merge commits) - [ ] Ran `clang-format` v8.0.0 (via `make format` or the Visual Studio extension) - [ ] Compiles - [ ] Ran all tests - [ ] If change impacts performance, include supporting evidence per the [performance document](https://github.com/stellar/stellar-core/blob/master/performance-eval/performance-eval.md)
stellar · Sep 17, 2024 · 7712e4a · 7712e4a
2 parents df57269 + 0a1c2f7
commit 7712e4a
Show file tree

Hide file tree

Showing 15 changed files with 1,936 additions and 1,057 deletions.
diff --git a/docs/software/commands.md b/docs/software/commands.md
@@ -19,6 +19,22 @@ Common options can be placed at any place in the command line.
 ## Command line options
 Command options can only by placed after command.
 
+* **apply-load**: Applies Soroban transactions by repeatedly generating transactions and closing
+them directly through the LedgerManager. The parameters specified below configure the network limits, and
+they're all required - **--ledger-max-instructions N**, **--ledger-max-read-entries N**, **--ledger-max-write-entries N**, **--ledger-max-read-byte N**, **--ledger-max-write-bytes N**, **--ledger-max-tx-size N**, **--ledger-max-tx-count N**. This command will generate enough transactions to fill up a synthetic transaction queue (it's just a list of transactions with the same limits as the real queue), and then create a transaction set off of that to
+apply.
+
+* At the moment, the Soroban transactions are generated using some of the same config parameters as the **generateload** command. Specifically,
+    `ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING=true`,
+    `LOADGEN_NUM_DATA_ENTRIES_FOR_TESTING`,
+    `LOADGEN_NUM_DATA_ENTRIES_DISTRIBUTION_FOR_TESTING`,
+    `LOADGEN_IO_KILOBYTES_FOR_TESTING`,
+    `LOADGEN_IO_KILOBYTES_DISTRIBUTION_FOR_TESTING`,
+    `LOADGEN_TX_SIZE_BYTES_FOR_TESTING`,
+    `LOADGEN_TX_SIZE_BYTES_DISTRIBUTION_FOR_TESTING`,
+    `LOADGEN_INSTRUCTIONS_FOR_TESTING`, and
+    `LOADGEN_INSTRUCTIONS_DISTRIBUTION_FOR_TESTING`.
+
 * **catchup <DESTINATION-LEDGER/LEDGER-COUNT>**: Perform catchup from history
   archives without connecting to network. For new instances (with empty history
   tables - only ledger 1 present in the database) it will respect LEDGER-COUNT

diff --git a/docs/stellar-core_example.cfg b/docs/stellar-core_example.cfg
@@ -494,7 +494,7 @@ MANUAL_CLOSE=false
 
 # ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING (true or false) defaults to false
 # Enables synthetic load generation on demand.
-# The load is triggered by the `generateload` runtime command.
+# The load is triggered by the `generateload` runtime command or the `apply-load` command line command.
 # This option only exists for stress-testing and should not be enabled in
 #  production networks.
 ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING=false

diff --git a/src/herder/HerderImpl.cpp b/src/herder/HerderImpl.cpp
@@ -61,7 +61,6 @@ constexpr uint32 const CLOSE_TIME_DRIFT_SECONDS_THRESHOLD = 10;
 constexpr uint32 const TRANSACTION_QUEUE_TIMEOUT_LEDGERS = 4;
 constexpr uint32 const TRANSACTION_QUEUE_BAN_LEDGERS = 10;
 constexpr uint32 const TRANSACTION_QUEUE_SIZE_MULTIPLIER = 2;
-constexpr uint32 const SOROBAN_TRANSACTION_QUEUE_SIZE_MULTIPLIER = 2;
 
 std::unique_ptr<Herder>
 Herder::create(Application& app)

diff --git a/src/herder/HerderImpl.h b/src/herder/HerderImpl.h
@@ -25,6 +25,8 @@ class Timer;
 
 namespace stellar
 {
+constexpr uint32 const SOROBAN_TRANSACTION_QUEUE_SIZE_MULTIPLIER = 2;
+
 class Application;
 class LedgerManager;
 class HerderSCPDriver;

diff --git a/src/main/CommandHandler.cpp b/src/main/CommandHandler.cpp
@@ -1405,7 +1405,8 @@ CommandHandler::generateLoad(std::string const& params, std::string& retStr)
         if (cfg.mode == LoadGenMode::SOROBAN_CREATE_UPGRADE)
         {
             auto configUpgradeKey =
-                mApp.getLoadGenerator().getConfigUpgradeSetKey(cfg);
+                mApp.getLoadGenerator().getConfigUpgradeSetKey(
+                    cfg.getSorobanUpgradeConfig());
             auto configUpgradeKeyStr = stellar::decoder::encode_b64(
                 xdr::xdr_to_opaque(configUpgradeKey));
             res["config_upgrade_set_key"] = configUpgradeKeyStr;

diff --git a/src/main/CommandLine.cpp b/src/main/CommandLine.cpp
@@ -27,9 +27,11 @@
 #include "main/SettingsUpgradeUtils.h"
 #include "main/StellarCoreVersion.h"
 #include "main/dumpxdr.h"
+#include "medida/metrics_registry.h"
 #include "overlay/OverlayManager.h"
 #include "rust/RustBridge.h"
 #include "scp/QuorumSetUtils.h"
+#include "simulation/ApplyLoad.h"
 #include "transactions/TransactionUtils.h"
 #include "util/Logging.h"
 #include "util/types.h"
@@ -1799,6 +1801,167 @@ runGenFuzz(CommandLineArgs const& args)
             return 0;
         });
 }
+
+int
+runApplyLoad(CommandLineArgs const& args)
+{
+    CommandLine::ConfigOption configOption;
+
+    uint64_t ledgerMaxInstructions = 0;
+    uint64_t ledgerMaxReadLedgerEntries = 0;
+    uint64_t ledgerMaxReadBytes = 0;
+    uint64_t ledgerMaxWriteLedgerEntries = 0;
+    uint64_t ledgerMaxWriteBytes = 0;
+    uint64_t ledgerMaxTxCount = 0;
+    uint64_t ledgerMaxTransactionsSizeBytes = 0;
+
+    ParserWithValidation ledgerMaxInstructionsParser{
+        clara::Opt(ledgerMaxInstructions,
+                   "LedgerMaxInstructions")["--ledger-max-instructions"]
+            .required(),
+        [&] {
+            return ledgerMaxInstructions > 0
+                       ? ""
+                       : "ledgerMaxInstructions must be > 0";
+        }};
+
+    ParserWithValidation ledgerMaxReadLedgerEntriesParser{
+        clara::Opt(ledgerMaxReadLedgerEntries,
+                   "LedgerMaxReadLedgerEntries")["--ledger-max-read-entries"]
+            .required(),
+        [&] {
+            return ledgerMaxReadLedgerEntries > 0
+                       ? ""
+                       : "ledgerMaxReadLedgerEntries must be > 0";
+        }};
+
+    ParserWithValidation ledgerMaxReadBytesParser{
+        clara::Opt(ledgerMaxReadBytes,
+                   "LedgerMaxReadBytes")["--ledger-max-read-bytes"]
+            .required(),
+        [&] {
+            return ledgerMaxReadBytes > 0 ? ""
+                                          : "ledgerMaxReadBytes must be > 0";
+        }};
+
+    ParserWithValidation ledgerMaxWriteLedgerEntriesParser{
+        clara::Opt(ledgerMaxWriteLedgerEntries,
+                   "LedgerMaxWriteLedgerEntries")["--ledger-max-write-entries"]
+            .required(),
+        [&] {
+            return ledgerMaxWriteLedgerEntries > 0
+                       ? ""
+                       : "ledgerMaxWriteLedgerEntries must be > 0";
+        }};
+
+    ParserWithValidation ledgerMaxWriteBytesParser{
+        clara::Opt(ledgerMaxWriteBytes,
+                   "LedgerMaxWriteBytes")["--ledger-max-write-bytes"]
+            .required(),
+        [&] {
+            return ledgerMaxWriteBytes > 0 ? ""
+                                           : "ledgerMaxWriteBytes must be > 0";
+        }};
+
+    ParserWithValidation ledgerMaxTxCountParser{
+        clara::Opt(ledgerMaxTxCount,
+                   "LedgerMaxTxCount")["--ledger-max-tx-count"]
+            .required(),
+        [&] {
+            return ledgerMaxTxCount > 0 ? "" : "ledgerMaxTxCount must be > 0";
+        }};
+
+    ParserWithValidation ledgerMaxTransactionsSizeBytesParser{
+        clara::Opt(ledgerMaxTransactionsSizeBytes,
+                   "LedgerMaxTransactionsSizeBytes")["--ledger-max-tx-size"]
+            .required(),
+        [&] {
+            return ledgerMaxTransactionsSizeBytes > 0
+                       ? ""
+                       : "ledgerMaxTransactionsSizeBytes must be > 0";
+        }};
+
+    return runWithHelp(
+        args,
+        {configurationParser(configOption), ledgerMaxInstructionsParser,
+         ledgerMaxReadLedgerEntriesParser, ledgerMaxReadBytesParser,
+         ledgerMaxWriteLedgerEntriesParser, ledgerMaxWriteBytesParser,
+         ledgerMaxTxCountParser, ledgerMaxTransactionsSizeBytesParser},
+        [&] {
+            auto config = configOption.getConfig();
+            config.RUN_STANDALONE = true;
+
+            VirtualClock clock(VirtualClock::REAL_TIME);
+            auto appPtr = Application::create(clock, config);
+
+            auto& app = *appPtr;
+            {
+                auto& lm = app.getLedgerManager();
+                app.start();
+
+                ApplyLoad al(app, ledgerMaxInstructions,
+                             ledgerMaxReadLedgerEntries, ledgerMaxReadBytes,
+                             ledgerMaxWriteLedgerEntries, ledgerMaxWriteBytes,
+                             ledgerMaxTxCount, ledgerMaxTransactionsSizeBytes);
+
+                auto& ledgerClose =
+                    app.getMetrics().NewTimer({"ledger", "ledger", "close"});
+                ledgerClose.Clear();
+
+                auto& cpuInsRatio = app.getMetrics().NewHistogram(
+                    {"soroban", "host-fn-op",
+                     "invoke-time-fsecs-cpu-insn-ratio"});
+                cpuInsRatio.Clear();
+
+                auto& cpuInsRatioExclVm = app.getMetrics().NewHistogram(
+                    {"soroban", "host-fn-op",
+                     "invoke-time-fsecs-cpu-insn-ratio-excl-vm"});
+                cpuInsRatioExclVm.Clear();
+
+                for (size_t i = 0; i < 20; ++i)
+                {
+                    al.benchmark();
+                }
+
+                CLOG_INFO(Perf, "Max ledger close: {} milliseconds",
+                          ledgerClose.max());
+                CLOG_INFO(Perf, "Min ledger close: {} milliseconds",
+                          ledgerClose.min());
+                CLOG_INFO(Perf, "Mean ledger close:  {} milliseconds",
+                          ledgerClose.mean());
+
+                CLOG_INFO(Perf, "Max CPU ins ratio: {}",
+                          cpuInsRatio.max() / 1000000);
+                CLOG_INFO(Perf, "Mean CPU ins ratio:  {}",
+                          cpuInsRatio.mean() / 1000000);
+
+                CLOG_INFO(Perf, "Max CPU ins ratio excl VM: {}",
+                          cpuInsRatioExclVm.max() / 1000000);
+                CLOG_INFO(Perf, "Mean CPU ins ratio excl VM:  {}",
+                          cpuInsRatioExclVm.mean() / 1000000);
+
+                CLOG_INFO(Perf, "Tx count utilization {}%",
+                          al.getTxCountUtilization().mean() / 1000.0);
+                CLOG_INFO(Perf, "Instruction utilization {}%",
+                          al.getInstructionUtilization().mean() / 1000.0);
+                CLOG_INFO(Perf, "Tx size utilization {}%",
+                          al.getTxSizeUtilization().mean() / 1000.0);
+                CLOG_INFO(Perf, "Read bytes utilization {}%",
+                          al.getReadByteUtilization().mean() / 1000.0);
+                CLOG_INFO(Perf, "Write bytes utilization {}%",
+                          al.getWriteByteUtilization().mean() / 1000.0);
+                CLOG_INFO(Perf, "Read entry utilization {}%",
+                          al.getReadEntryUtilization().mean() / 1000.0);
+                CLOG_INFO(Perf, "Write entry utilization {}%",
+                          al.getWriteEntryUtilization().mean() / 1000.0);
+
+                CLOG_INFO(Perf, "Tx Success Rate: {:f}%",
+                          al.successRate() * 100);
+            }
+
+            return 0;
+        });
+}
 #endif
 
 int
@@ -1871,6 +2034,7 @@ handleCommandLine(int argc, char* const* argv)
          {"fuzz", "run a single fuzz input and exit", runFuzz},
          {"gen-fuzz", "generate a random fuzzer input file", runGenFuzz},
          {"test", "execute test suite", runTest},
+         {"apply-load", "run apply time load test", runApplyLoad},
 #endif
          {"version", "print version information", runVersion}}};