path: root/0001-Limit-job-execution-dependant-on-available-memory-m.patch

From ff2fa001eb1ad5e24850eacc4cbed66ecf038de0 Mon Sep 17 00:00:00 2001
From: Fredrick Brennan <copypaste@kittens.ph>
Date: Mon, 20 Mar 2023 20:07:19 -0400
Subject: [PATCH] Limit job execution dependant on available memory (-m)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: bartus <arch-user-repoᘓbartus.33mail.com>
---
 doc/manual.asciidoc |  9 +++++++
 src/build.cc        |  4 ++-
 src/build.h         |  7 ++++-
 src/ninja.cc        | 14 +++++++++-
 src/util.cc         | 64 +++++++++++++++++++++++++++++++++++++++++++++
 src/util.h          |  6 ++++-
 src/util_test.cc    | 30 +++++++++++++++++++++
 7 files changed, 130 insertions(+), 4 deletions(-)

diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc
index 659c68b1..a914c403 100644
--- a/doc/manual.asciidoc
+++ b/doc/manual.asciidoc
@@ -187,6 +187,15 @@ match those of Make; e.g `ninja -C build -j 20` changes into the
 Ninja defaults to running commands in parallel anyway, so typically
 you don't need to pass `-j`.)
 
+In certain environments it might be helpful to dynamically limit the
+amount of running jobs depending on the currently available resources.
+Thus `ninja` can be started with `-l` and/or `-m` options to prevent
+from starting new jobs in case load average or memory usage exceed
+the defined limit. `-l` takes one single numeric argument defining
+the limit for the typical unix load average. `-m` takes one single
+numeric argument within [0,100] as a percentage of the memory usage
+(reduced by caches) on the host.
+
 
 Environment variables
 ~~~~~~~~~~~~~~~~~~~~~
diff --git a/src/build.cc b/src/build.cc
index 6f11ed7a..9dd0eb69 100644
--- a/src/build.cc
+++ b/src/build.cc
@@ -478,7 +478,9 @@ bool RealCommandRunner::CanRunMore() const {
       subprocs_.running_.size() + subprocs_.finished_.size();
   return (int)subproc_number < config_.parallelism
     && ((subprocs_.running_.empty() || config_.max_load_average <= 0.0f)
-        || GetLoadAverage() < config_.max_load_average);
+        || GetLoadAverage() < config_.max_load_average)
+    && ((subprocs_.running_.empty() || config_.max_memory_usage <= 0.0f)
+       || GetMemoryUsage() < config_.max_memory_usage);
 }
 
 bool RealCommandRunner::StartCommand(Edge* edge) {
diff --git a/src/build.h b/src/build.h
index d697dfb8..5748c70b 100644
--- a/src/build.h
+++ b/src/build.h
@@ -156,7 +156,8 @@ struct CommandRunner {
 /// Options (e.g. verbosity, parallelism) passed to a build.
 struct BuildConfig {
   BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1),
-                  failures_allowed(1), max_load_average(-0.0f) {}
+                  failures_allowed(1), max_load_average(-0.0f),
+                  max_memory_usage(-0.0f) {}
 
   enum Verbosity {
     QUIET,  // No output -- used when testing.
@@ -171,6 +172,10 @@ struct BuildConfig {
   /// The maximum load average we must not exceed. A negative value
   /// means that we do not have any limit.
   double max_load_average;
+  /// The maximum memory usage we must not exceed. The value is
+  /// defined within [0.0,1.0]. A negative values indicates that we do
+  /// not have any limit.
+  double max_memory_usage;
   DepfileParserOptions depfile_parser_options;
 };
 
diff --git a/src/ninja.cc b/src/ninja.cc
index 2b71eb17..c92515b8 100644
--- a/src/ninja.cc
+++ b/src/ninja.cc
@@ -229,6 +229,10 @@ void Usage(const BuildConfig& config) {
 "  -j N     run N jobs in parallel (0 means infinity) [default=%d on this system]\n"
 "  -k N     keep going until N jobs fail (0 means infinity) [default=1]\n"
 "  -l N     do not start new jobs if the load average is greater than N\n"
+"  -m N     do not start new jobs if the memory usage exceeds N percent\n"
+#if !(defined(__APPLE__) || defined(linux) || defined(_WIN32))
+"           (not yet implemented on this platform)\n"
+#endif
 "  -n       dry run (don't run commands but act like they succeeded)\n"
 "\n"
 "  -d MODE  enable debugging (use '-d list' to list modes)\n"
@@ -1428,7 +1432,7 @@ int ReadFlags(int* argc, char*** argv,
 
   int opt;
   while (!options->tool &&
-         (opt = getopt_long(*argc, *argv, "d:f:j:k:l:nt:vw:C:h", kLongOptions,
+         (opt = getopt_long(*argc, *argv, "d:f:j:k:l:m:nt:vw:C:h", kLongOptions,
                             NULL)) != -1) {
     switch (opt) {
       case 'd':
@@ -1470,6 +1474,14 @@ int ReadFlags(int* argc, char*** argv,
         config->max_load_average = value;
         break;
       }
+      case 'm': {
+        char* end;
+        const int value = strtol(optarg, &end, 10);
+        if (end == optarg || value < 0 || value > 100)
+          Fatal("-m parameter is invalid: allowed range is [0,100]");
+        config->max_memory_usage = value/100.0; // map to [0.0,100.0]
+        break;
+      }
       case 'n':
         config->dry_run = true;
         break;
diff --git a/src/util.cc b/src/util.cc
index ef5f1033..2c754bc4 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -19,6 +19,7 @@
 #include <io.h>
 #elif defined( _WIN32)
 #include <windows.h>
+#include <Psapi.h>
 #include <io.h>
 #include <share.h>
 #endif
@@ -40,6 +41,7 @@
 
 #include <vector>
 
+// to determine the load average
 #if defined(__APPLE__) || defined(__FreeBSD__)
 #include <sys/sysctl.h>
 #elif defined(__SVR4) && defined(__sun)
@@ -58,6 +60,13 @@
 #include <sys/cpuset.h>
 #endif
 
+// to determine the memory usage
+#if defined(__APPLE__)
+#include <mach/mach.h>
+#elif defined(linux)
+#include <fstream>
+#endif
+
 #include "edit_distance.h"
 
 using namespace std;
@@ -830,6 +839,61 @@ double GetLoadAverage() {
 }
 #endif // _WIN32
 
+double GetMemoryUsage() {
+#if defined(__APPLE__)
+  // total memory
+  uint64_t physical_memory;
+  {
+    size_t length = sizeof(physical_memory);
+    if (!(sysctlbyname("hw.memsize",
+                       &physical_memory, &length, NULL, 0) == 0)) {
+      return -0.0f;
+    }
+  }
+
+  // pagesize
+  unsigned pagesize = 0;
+  {
+    size_t length = sizeof(pagesize);
+    if (!(sysctlbyname("hw.pagesize",
+                       &pagesize, &length, NULL, 0) == 0)) {
+      return -0.0f;
+    }
+  }
+
+  // current free memory
+  vm_statistics_data_t vm;
+  mach_msg_type_number_t ic = HOST_VM_INFO_COUNT;
+  host_statistics(mach_host_self(), HOST_VM_INFO, (host_info_t) &vm, &ic);
+
+  return 1.0 - static_cast<double>(pagesize) * vm.free_count / physical_memory;
+#elif defined(linux)
+  ifstream meminfo("/proc/meminfo", ifstream::in);
+  string token;
+  uint64_t free(0), total(0);
+  while (meminfo >> token) {
+    if (token == "MemAvailable:") {
+      meminfo >> free;
+    } else if (token == "MemTotal:") {
+      meminfo >> total;
+    } else {
+      continue;
+    }
+    if (free > 0 && total > 0) {
+      return (double) (total - free) / total;
+    }
+  }
+  return -0.0f; // this is the fallback in case the API has changed
+#elif (_WIN32)
+  PERFORMANCE_INFORMATION perf;
+  GetPerformanceInfo(&perf, sizeof(PERFORMANCE_INFORMATION));
+  return 1.0 - static_cast<double>(perf.PhysicalAvailable) /
+               static_cast<double>(perf.PhysicalTotal);
+#else // any unsupported platform
+  return -0.0f;
+#endif
+}
+
 string ElideMiddle(const string& str, size_t width) {
   switch (width) {
       case 0: return "";
diff --git a/src/util.h b/src/util.h
index 4a7fea22..20a3018e 100644
--- a/src/util.h
+++ b/src/util.h
@@ -100,9 +100,13 @@ std::string StripAnsiEscapeCodes(const std::string& in);
 int GetProcessorCount();
 
 /// @return the load average of the machine. A negative value is returned
-/// on error.
+/// on error or if the feature is not supported on this platform.
 double GetLoadAverage();
 
+/// @return the memory usage of the machine. A negative value is returned
+/// on error or if the feature is not supported on this platform.
+double GetMemoryUsage();
+
 /// Elide the given string @a str with '...' in the middle if the length
 /// exceeds @a width.
 std::string ElideMiddle(const std::string& str, size_t width);
diff --git a/src/util_test.cc b/src/util_test.cc
index d58b1708..157da34d 100644
--- a/src/util_test.cc
+++ b/src/util_test.cc
@@ -17,6 +17,12 @@
 #include "test.h"
 
 using namespace std;
+#ifdef _WIN32
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#else
+#include <unistd.h>
+#endif
 
 namespace {
 
@@ -408,6 +414,30 @@ TEST(StripAnsiEscapeCodes, StripColors) {
             stripped);
 }
 
+TEST(SystemInformation, ProcessorCount) {
+#ifdef _WIN32
+  SYSTEM_INFO info;
+  GetSystemInfo(&info);
+  const int expected = info.dwNumberOfProcessors;
+#else
+  const int expected = sysconf(_SC_NPROCESSORS_ONLN);
+#endif
+  EXPECT_EQ(expected, GetProcessorCount());
+}
+
+TEST(SystemInformation, LoadAverage) {
+#if ! (defined(_WIN32) || defined(__CYGWIN__))
+  EXPECT_LT(0.0f, GetLoadAverage());
+#endif
+}
+
+TEST(SystemInformation, MemoryUsage) {
+#if defined(__APPLE__) || defined(linux) || defined(_WIN32)
+  EXPECT_LT(0.0f, GetMemoryUsage());
+  EXPECT_GT(1.0f, GetMemoryUsage());
+#endif
+}
+
 TEST(ElideMiddle, NothingToElide) {
   string input = "Nothing to elide in this short string.";
   EXPECT_EQ(input, ElideMiddle(input, 80));
-- 
2.40.0