lower default values for small systems (arangodb#11485)

dcosta72 · Apr 22, 2020 · 7d6b289 · 7d6b289
1 parent 91dd1c7
commit 7d6b289
Show file tree

Hide file tree

Showing 18 changed files with 200 additions and 52 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,5 +1,14 @@
 devel
 -----
+
+* Lower the default values for `--rocksdb.block-cache-size` and 
+  `--rocksdb.total-write-buffer-size` for systems with less than 1 GB of RAM.
+  For these systems the default value were presumably too high and needed manual
+  adjustment.
+
+* Allow to override the detected number of avaiable CPU cores via an environment
+  variable ARANGODB_OVERRIDE_DETECTED_NUMBER_OF_CORES.
+
 * Fix a bug in the optimizer which handled combinations of smart joins with
   query components that had to run on a coordinator wrong.
 

diff --git a/arangod/Cluster/MaintenanceFeature.cpp b/arangod/Cluster/MaintenanceFeature.cpp
@@ -30,6 +30,7 @@
 #include "ApplicationFeatures/ApplicationServer.h"
 #include "Basics/ConditionLocker.h"
 #include "Basics/MutexLocker.h"
+#include "Basics/NumberOfCores.h"
 #include "Basics/ReadLocker.h"
 #include "Basics/StaticStrings.h"
 #include "Basics/WriteLocker.h"
@@ -93,7 +94,7 @@ void MaintenanceFeature::init() {
 
   _maintenanceThreadsMax =
       (std::max)(static_cast<uint32_t>(minThreadLimit),
-                 static_cast<uint32_t>(TRI_numberProcessors() / 4 + 1));
+                 static_cast<uint32_t>(NumberOfCores::getValue() / 4 + 1));
   _secondsActionsBlock = 2;
   _secondsActionsLinger = 3600;
 }  // MaintenanceFeature::init

diff --git a/arangod/GeneralServer/GeneralServerFeature.cpp b/arangod/GeneralServer/GeneralServerFeature.cpp
@@ -30,6 +30,7 @@
 #include "Agency/RestAgencyPrivHandler.h"
 #include "ApplicationFeatures/HttpEndpointProvider.h"
 #include "Aql/RestAqlHandler.h"
+#include "Basics/NumberOfCores.h"
 #include "Basics/StringUtils.h"
 #include "Basics/application-exit.h"
 #include "Cluster/AgencyCallbackRegistry.h"
@@ -136,7 +137,7 @@ GeneralServerFeature::GeneralServerFeature(application_features::ApplicationServ
   startsAfter<UpgradeFeature>();
 
   _numIoThreads = (std::max)(static_cast<uint64_t>(1),
-                             static_cast<uint64_t>(TRI_numberProcessors() / 4));
+                             static_cast<uint64_t>(NumberOfCores::getValue() / 4));
   if (_numIoThreads > _maxIoThreads) {
     _numIoThreads = _maxIoThreads;
   }

diff --git a/arangod/IResearch/IResearchFeature.cpp b/arangod/IResearch/IResearchFeature.cpp
@@ -35,6 +35,7 @@
 #include "Aql/Function.h"
 #include "Aql/Functions.h"
 #include "Basics/ConditionLocker.h"
+#include "Basics/NumberOfCores.h"
 #include "Cluster/ClusterFeature.h"
 #include "Cluster/ClusterInfo.h"
 #include "Cluster/ServerState.h"
@@ -276,7 +277,7 @@ size_t computeThreadPoolSize(size_t threads, size_t threadsLimit) {
   return threads ? threads
                  : std::max(MIN_THREADS,
                             std::min(maxThreads,
-                                     size_t(std::thread::hardware_concurrency()) / 4));
+                                     arangodb::NumberOfCores::getValue() / 4));
 }
 
 bool upgradeSingleServerArangoSearchView0_1(

diff --git a/arangod/Pregel/PregelFeature.cpp b/arangod/Pregel/PregelFeature.cpp
@@ -26,6 +26,7 @@
 
 #include "ApplicationFeatures/ApplicationServer.h"
 #include "Basics/MutexLocker.h"
+#include "Basics/NumberOfCores.h"
 #include "Cluster/ClusterFeature.h"
 #include "Cluster/ClusterInfo.h"
 #include "Cluster/ServerState.h"
@@ -234,7 +235,7 @@ PregelFeature::~PregelFeature() {
 std::shared_ptr<PregelFeature> PregelFeature::instance() { return ::instance; }
 
 size_t PregelFeature::availableParallelism() {
-  const size_t procNum = TRI_numberProcessors();
+  const size_t procNum = NumberOfCores::getValue();
   return procNum < 1 ? 1 : procNum;
 }
 

diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp
@@ -26,6 +26,7 @@
 #include "ApplicationFeatures/ApplicationServer.h"
 #include "Basics/Exceptions.h"
 #include "Basics/FileUtils.h"
+#include "Basics/NumberOfCores.h"
 #include "Basics/ReadLocker.h"
 #include "Basics/Result.h"
 #include "Basics/RocksDBLogger.h"
@@ -419,7 +420,7 @@ void RocksDBEngine::start() {
 
   rocksdb::TransactionDBOptions transactionOptions;
   // number of locks per column_family
-  transactionOptions.num_stripes = TRI_numberProcessors();
+  transactionOptions.num_stripes = NumberOfCores::getValue();
   transactionOptions.transaction_lock_timeout = opts._transactionLockTimeout;
 
   _options.allow_fallocate = opts._allowFAllocate;

diff --git a/arangod/Scheduler/SchedulerFeature.cpp b/arangod/Scheduler/SchedulerFeature.cpp
@@ -29,6 +29,7 @@
 
 #include "ApplicationFeatures/ApplicationServer.h"
 #include "ApplicationFeatures/GreetingsFeaturePhase.h"
+#include "Basics/NumberOfCores.h"
 #include "Basics/application-exit.h"
 #include "Basics/signals.h"
 #include "Basics/system-functions.h"
@@ -59,7 +60,7 @@ namespace {
 /// @brief return the default number of threads to use (upper bound)
 size_t defaultNumberOfThreads() {
   // use two times the number of hardware threads as the default
-  size_t result = TRI_numberProcessors() * 2;
+  size_t result = arangodb::NumberOfCores::getValue() * 2;
   // but only if higher than 64. otherwise use a default minimum value of 64
   if (result < 64) {
     result = 64;
@@ -125,7 +126,7 @@ void SchedulerFeature::collectOptions(std::shared_ptr<options::ProgramOptions> o
 }
 
 void SchedulerFeature::validateOptions(std::shared_ptr<options::ProgramOptions> options) {
-  auto const N = TRI_numberProcessors();
+  auto const N = NumberOfCores::getValue();
 
   LOG_TOPIC("2ef39", DEBUG, arangodb::Logger::THREADS)
       << "Detected number of processors: " << N;

diff --git a/arangod/StorageEngine/RocksDBOptionFeature.cpp b/arangod/StorageEngine/RocksDBOptionFeature.cpp
@@ -28,6 +28,7 @@
 #include "RocksDBOptionFeature.h"
 
 #include "ApplicationFeatures/ApplicationServer.h"
+#include "Basics/NumberOfCores.h"
 #include "Basics/PhysicalMemory.h"
 #include "Basics/application-exit.h"
 #include "Basics/process-utils.h"
@@ -52,6 +53,37 @@ namespace {
 rocksdb::TransactionDBOptions rocksDBTrxDefaults;
 rocksdb::Options rocksDBDefaults;
 rocksdb::BlockBasedTableOptions rocksDBTableOptionsDefaults;
+
+uint64_t defaultBlockCacheSize() {
+  if (PhysicalMemory::getValue() >= (static_cast<uint64_t>(4) << 30)) {
+    // if we have at least 4GB of RAM, the default size is (RAM - 2GB) * 0.3 
+    return static_cast<uint64_t>((PhysicalMemory::getValue() - (static_cast<uint64_t>(2) << 30)) * 0.3);
+  }
+  if (PhysicalMemory::getValue() >= (static_cast<uint64_t>(2) << 30)) {
+    // if we have at least 2GB of RAM, the default size is 512MB
+    return (static_cast<uint64_t>(512) << 20);
+  }
+  if (PhysicalMemory::getValue() >= (static_cast<uint64_t>(1) << 30)) {
+    // if we have at least 1GB of RAM, the default size is 256MB
+    return (static_cast<uint64_t>(256) << 20);
+  }
+  // for everything else the default size is 128MB
+  return (static_cast<uint64_t>(128) << 20);
+}
+
+uint64_t defaultTotalWriteBufferSize() {
+  if (PhysicalMemory::getValue() >= (static_cast<uint64_t>(4) << 30)) {
+    // if we have at least 4GB of RAM, the default size is (RAM - 2GB) * 0.4 
+    return static_cast<uint64_t>((PhysicalMemory::getValue() - (static_cast<uint64_t>(2) << 30)) * 0.4);
+  } 
+  if (PhysicalMemory::getValue() >= (static_cast<uint64_t>(1) << 30)) {
+    // if we have at least 1GB of RAM, the default size is 512MB
+    return (static_cast<uint64_t>(512) << 20);
+  }
+  // for everything else the default size is 256MB
+  return (static_cast<uint64_t>(256) << 20);
+}
+
 }  // namespace
 
 RocksDBOptionFeature::RocksDBOptionFeature(application_features::ApplicationServer& server)
@@ -73,10 +105,7 @@ RocksDBOptionFeature::RocksDBOptionFeature(application_features::ApplicationServ
       _numThreadsLow(0),
       _targetFileSizeBase(rocksDBDefaults.target_file_size_base),
       _targetFileSizeMultiplier(rocksDBDefaults.target_file_size_multiplier),
-      _blockCacheSize((PhysicalMemory::getValue() >= (static_cast<uint64_t>(4) << 30))
-                          ? static_cast<uint64_t>(
-                                ((PhysicalMemory::getValue() - (static_cast<uint64_t>(2) << 30)) * 0.3))
-                          : (256 << 20)),
+      _blockCacheSize(::defaultBlockCacheSize()),
       _blockCacheShardBits(-1),
       _tableBlockSize(
           std::max(rocksDBTableOptionsDefaults.block_size,
@@ -102,7 +131,7 @@ RocksDBOptionFeature::RocksDBOptionFeature(application_features::ApplicationServ
       _exclusiveWrites(false) {
   // setting the number of background jobs to
   _maxBackgroundJobs = static_cast<int32_t>(
-      std::max((size_t)2, std::min(TRI_numberProcessors(), (size_t)8)));
+      std::max(static_cast<size_t>(2), std::min(NumberOfCores::getValue(), static_cast<size_t>(8))));
 #ifdef _WIN32
   // Windows code does not (yet) support lowering thread priority of
   //  compactions.  Therefore it is possible for rocksdb to use all
@@ -115,12 +144,7 @@ RocksDBOptionFeature::RocksDBOptionFeature(application_features::ApplicationServ
 
   if (_totalWriteBufferSize == 0) {
     // unlimited write buffer size... now set to some fraction of physical RAM
-    if (PhysicalMemory::getValue() >= (static_cast<uint64_t>(4) << 30)) {
-      _totalWriteBufferSize = static_cast<uint64_t>(
-          (PhysicalMemory::getValue() - (static_cast<uint64_t>(2) << 30)) * 0.4);
-    } else {
-      _totalWriteBufferSize = (512 << 20);
-    }
+    _totalWriteBufferSize = ::defaultTotalWriteBufferSize();
   }
 
   setOptional(true);
@@ -419,7 +443,7 @@ void RocksDBOptionFeature::validateOptions(std::shared_ptr<ProgramOptions> optio
 
 void RocksDBOptionFeature::start() {
   uint32_t max = _maxBackgroundJobs / 2;
-  uint32_t clamped = std::max(std::min((uint32_t)TRI_numberProcessors(), max), 1U);
+  uint32_t clamped = std::max(std::min(static_cast<uint32_t>(NumberOfCores::getValue()), max), 1U);
   // lets test this out
   if (_numThreadsHigh == 0) {
     _numThreadsHigh = clamped;

diff --git a/arangosh/Dump/DumpFeature.cpp b/arangosh/Dump/DumpFeature.cpp
@@ -35,6 +35,7 @@
 #include "ApplicationFeatures/ApplicationServer.h"
 #include "Basics/FileUtils.h"
 #include "Basics/MutexLocker.h"
+#include "Basics/NumberOfCores.h"
 #include "Basics/Result.h"
 #include "Basics/ScopeGuard.h"
 #include "Basics/StaticStrings.h"
@@ -709,7 +710,7 @@ void DumpFeature::validateOptions(std::shared_ptr<options::ProgramOptions> optio
 
   uint32_t clamped =
       boost::algorithm::clamp(_options.threadCount, 1,
-                              4 * static_cast<uint32_t>(TRI_numberProcessors()));
+                              4 * static_cast<uint32_t>(NumberOfCores::getValue()));
   if (_options.threadCount != clamped) {
     LOG_TOPIC("0460e", WARN, Logger::DUMP) << "capping --threads value to " << clamped;
     _options.threadCount = clamped;

diff --git a/arangosh/Import/ImportFeature.cpp b/arangosh/Import/ImportFeature.cpp
@@ -24,6 +24,7 @@
 
 #include "ApplicationFeatures/ApplicationServer.h"
 #include "Basics/FileUtils.h"
+#include "Basics/NumberOfCores.h"
 #include "Basics/StringUtils.h"
 #include "Basics/application-exit.h"
 #include "Basics/system-functions.h"
@@ -222,12 +223,12 @@ void ImportFeature::validateOptions(std::shared_ptr<options::ProgramOptions> opt
     LOG_TOPIC("9e3f9", WARN, arangodb::Logger::FIXME) << "capping --threads value to " << 1;
     _threadCount = 1;
   }
-  if (_threadCount > TRI_numberProcessors() * 2) {
+  if (_threadCount > NumberOfCores::getValue() * 2) {
     // it's not sensible to use just one thread ...
     //  and import's CPU usage is negligible, real limit is cluster cores
     LOG_TOPIC("aca46", WARN, arangodb::Logger::FIXME)
-        << "capping --threads value to " << TRI_numberProcessors() * 2;
-    _threadCount = (uint32_t)TRI_numberProcessors() * 2;
+        << "capping --threads value to " << NumberOfCores::getValue() * 2;
+    _threadCount = static_cast<uint32_t>(NumberOfCores::getValue()) * 2;
   }
 
   for (auto const& it : _translations) {

diff --git a/arangosh/Restore/RestoreFeature.cpp b/arangosh/Restore/RestoreFeature.cpp
@@ -35,6 +35,7 @@
 
 #include "ApplicationFeatures/ApplicationServer.h"
 #include "Basics/FileUtils.h"
+#include "Basics/NumberOfCores.h"
 #include "Basics/Result.h"
 #include "Basics/StaticStrings.h"
 #include "Basics/StringUtils.h"
@@ -1402,7 +1403,7 @@ void RestoreFeature::validateOptions(std::shared_ptr<options::ProgramOptions> op
   }
 
   auto clamped = boost::algorithm::clamp(_options.threadCount, uint32_t(1),
-                                         uint32_t(4 * TRI_numberProcessors()));
+                                         uint32_t(4 * NumberOfCores::getValue()));
   if (_options.threadCount != clamped) {
     LOG_TOPIC("53570", WARN, Logger::RESTORE) << "capping --threads value to " << clamped;
     _options.threadCount = clamped;

diff --git a/lib/ApplicationFeatures/EnvironmentFeature.cpp b/lib/ApplicationFeatures/EnvironmentFeature.cpp
@@ -31,6 +31,7 @@
 #include "ApplicationFeatures/GreetingsFeaturePhase.h"
 #include "ApplicationFeatures/MaxMapCountFeature.h"
 #include "Basics/FileUtils.h"
+#include "Basics/NumberOfCores.h"
 #include "Basics/PhysicalMemory.h"
 #include "Basics/Result.h"
 #include "Basics/StringUtils.h"
@@ -246,14 +247,13 @@ void EnvironmentFeature::prepare() {
     // file not found or value not convertible into integer
   }
 
-  // Report memory found:
-  uint64_t ram = PhysicalMemory::getValue();
-  std::string overriddenmsg;
-  if (PhysicalMemory::overridden()) {
-    overriddenmsg = " (overridden by environment variable)";
-  }
+  // Report memory and CPUs found:
   LOG_TOPIC("25362", INFO, Logger::MEMORY)
-  << "Available physical memory: " << ram << overriddenmsg << " bytes";
+    << "Available physical memory: " 
+    << PhysicalMemory::getValue() << " bytes" 
+    << (PhysicalMemory::overridden() ? " (overriden by environment variable)" : "")
+    << ", available cores: " << NumberOfCores::getValue()
+    << (NumberOfCores::overridden() ? " (overriden by environment variable)" : "");
 
   // test local ipv6 support
   try {

diff --git a/lib/ApplicationFeatures/MaxMapCountFeature.cpp b/lib/ApplicationFeatures/MaxMapCountFeature.cpp
@@ -24,6 +24,7 @@
 #include "ApplicationFeatures/ApplicationServer.h"
 #include "ApplicationFeatures/GreetingsFeaturePhase.h"
 #include "Basics/FileUtils.h"
+#include "Basics/NumberOfCores.h"
 #include "Basics/StringUtils.h"
 #include "Basics/Thread.h"
 #include "Basics/process-utils.h"
@@ -75,7 +76,7 @@ uint64_t MaxMapCountFeature::minimumExpectedMaxMappings() {
 #ifdef __linux__
   uint64_t expected = 65530;  // Linux kernel default
 
-  uint64_t nproc = TRI_numberProcessors();
+  uint64_t nproc = NumberOfCores::getValue();
 
   // we expect at most 8 times the number of cores as the effective number of
   // threads, and we want to allow at least 8000 mmaps per thread