From 56d8f86e389b8538ea5021e9f2acebb5891a5440 Mon Sep 17 00:00:00 2001
From: Timo Pollmeier <timo.pollmeier@greenbone.net>
Date: Mon, 22 Jul 2024 14:28:16 +0200
Subject: [PATCH] Add: New --min-mem-feed-update option

A new option is added which will make the automatic feed update wait
until a minimum amount of physical memory is available.

Additionally, the --mem-wait-retries can be used to set the number of
retries waiting for memory to be available in each process.
---
 doc/gvmd.8     |   6 +++
 doc/gvmd.8.xml |  18 +++++++
 doc/gvmd.html  |  18 +++++++
 src/gvmd.c     |  18 +++++++
 src/manage.c   | 143 ++++++++++++++++++++++++++++++++++++++++++++++++-
 src/manage.h   |  15 ++++++
 src/utils.c    |  23 ++++++++
 src/utils.h    |   6 +++
 8 files changed, 245 insertions(+), 2 deletions(-)

diff --git a/doc/gvmd.8 b/doc/gvmd.8
index 7972955c6..7179be5d4 100644
--- a/doc/gvmd.8
+++ b/doc/gvmd.8
@@ -121,9 +121,15 @@ Maximum size of user-defined message text in alert emails, in bytes.
 \fB--max-ips-per-target=\fINUMBER\fB\f1
 Maximum number of IPs per target.
 .TP
+\fB--mem-wait-retries=\fINUMBER\fB\f1
+How often to try waiting for available memory. Default: 30. Each retry will wait for 10 seconds. 
+.TP
 \fB-m, --migrate\f1
 Migrate the database and exit.
 .TP
+\fB--min-mem-feed-update=\fINUMBER\fB\f1
+Minimum memory in MiB for feed updates. Default: 0. Feed updates are skipped if less physical memory is available. 
+.TP
 \fB--modify-scanner=\fISCANNER-UUID\fB\f1
 Modify scanner SCANNER-UUID and exit.
 .TP
diff --git a/doc/gvmd.8.xml b/doc/gvmd.8.xml
index 7c2165808..8e4bca59a 100644
--- a/doc/gvmd.8.xml
+++ b/doc/gvmd.8.xml
@@ -287,12 +287,30 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
         <p>Maximum number of IPs per target.</p>
       </optdesc>
     </option>
+    <option>
+      <p><opt>--mem-wait-retries=<arg>NUMBER</arg></opt></p>
+      <optdesc>
+        <p>
+          How often to try waiting for available memory. Default: 30.
+          Each retry will wait for 10 seconds.
+        </p>
+      </optdesc>
+    </option>
     <option>
       <p><opt>-m, --migrate</opt></p>
       <optdesc>
         <p>Migrate the database and exit.</p>
       </optdesc>
     </option>
+    <option>
+      <p><opt>--min-mem-feed-update=<arg>NUMBER</arg></opt></p>
+      <optdesc>
+        <p>
+          Minimum memory in MiB for feed updates. Default: 0.
+          Feed updates are skipped if less physical memory is available.
+        </p>
+      </optdesc>
+    </option>
     <option>
       <p><opt>--modify-scanner=<arg>SCANNER-UUID</arg></opt></p>
       <optdesc>
diff --git a/doc/gvmd.html b/doc/gvmd.html
index 68913b68a..60ba46fe4 100644
--- a/doc/gvmd.html
+++ b/doc/gvmd.html
@@ -242,12 +242,30 @@ <h2>Options</h2>
       
     
     
+      <p><b>--mem-wait-retries=<em>NUMBER</em></b></p>
+      
+        <p>
+          How often to try waiting for available memory. Default: 30.
+           Each retry will wait for 10 seconds.
+        </p>
+      
+    
+    
       <p><b>-m, --migrate</b></p>
       
         <p>Migrate the database and exit.</p>
       
     
     
+      <p><b>--min-mem-feed-update=<em>NUMBER</em></b></p>
+      
+        <p>
+          Minimum memory in MiB for feed updates. Default: 0.
+          Feed updates are skipped if less physical memory is available.
+        </p>
+      
+    
+    
       <p><b>--modify-scanner=<em>SCANNER-UUID</em></b></p>
       
         <p>Modify scanner SCANNER-UUID and exit.</p>
diff --git a/src/gvmd.c b/src/gvmd.c
index 4ffdab4e8..05bc8c43c 100644
--- a/src/gvmd.c
+++ b/src/gvmd.c
@@ -1895,6 +1895,8 @@ gvmd (int argc, char** argv, char *env[])
   static gchar *broker_address = NULL;
   static gchar *feed_lock_path = NULL;
   static int feed_lock_timeout = 0;
+  static int mem_wait_retries = 30;
+  static int min_mem_feed_update = 0;
   static int vt_ref_insert_size = VT_REF_INSERT_SIZE_DEFAULT;
   static int vt_sev_insert_size = VT_SEV_INSERT_SIZE_DEFAULT;
   static gchar *vt_verification_collation = NULL;
@@ -2088,10 +2090,20 @@ gvmd (int argc, char** argv, char *env[])
           &max_ips_per_target,
           "Maximum number of IPs per target.",
           "<number>" },
+        { "mem-wait-retries", '\0', 0, G_OPTION_ARG_INT,
+          &mem_wait_retries,
+          "How often to try waiting for available memory. Default: 30."
+          " Each retry will wait for 10 seconds.",
+          "<number>" },
         { "migrate", 'm', 0, G_OPTION_ARG_NONE,
           &migrate_database,
           "Migrate the database and exit.",
           NULL },
+        { "min-mem-feed-update", '\0', 0, G_OPTION_ARG_INT,
+          &min_mem_feed_update,
+          "Minimum memory in MiB for feed updates. Default: 0."
+          " Feed updates are skipped if less physical memory is available.",
+          "<number>" },
         { "modify-scanner", '\0', 0, G_OPTION_ARG_STRING,
           &modify_scanner,
           "Modify scanner <scanner-uuid> and exit.",
@@ -2447,6 +2459,12 @@ gvmd (int argc, char** argv, char *env[])
       }
   }
 
+  /* Set number of retries waiting for memory */
+  set_mem_wait_retries (mem_wait_retries);
+  
+  /* Set minimum memory for feed updates */
+  set_min_mem_feed_update (min_mem_feed_update);
+
   /* Set relay mapper */
   if (relay_mapper)
     {
diff --git a/src/manage.c b/src/manage.c
index 0af8a5486..1465ca279 100644
--- a/src/manage.c
+++ b/src/manage.c
@@ -184,6 +184,16 @@ static gchar *feed_lock_path = NULL;
  */
 static int feed_lock_timeout = 0;
 
+/**
+ * @brief Retries for waiting for memory to be available.
+ */
+static int mem_wait_retries = 0;
+
+/**
+ * @brief Minimum available memory in MiB for running a feed update.
+ */
+static int min_mem_feed_update = 0;
+
 /**
  * @brief Path to the relay mapper executable, NULL to disable relays.
  */
@@ -5050,7 +5060,52 @@ feed_sync_required ()
   return FALSE;
 }
 
+/**
+ * @brief Wait for memory
+ * 
+ * @param[in]  check_func  Function to check memory, should return 1 if enough.
+ * @param[in]  retries     Number of retries.
+ * @param[in]  min_mem     Minimum memory in MiB, for logging only
+ * @param[in]  action      Short descriptor of action waiting for memory.
+ * 
+ * @return 0 if enough memory is available, 1 gave up
+ */
+static int
+wait_for_mem (int check_func(),
+              int retries,
+              int min_mem,
+              const char *action)
+{
+  int retry_number = 0;
+  while (check_func () == 0)
+    {
+      if (retry_number == 0)
+        {
+          g_info ("%s: not enough memory for %s"
+                  " (%lld / %d) MiB",
+                  __func__,
+                  action,
+                  phys_mem_available () / 1048576llu,
+                  min_mem);
+        }
+      else
+        {
+          g_debug ("%s: waiting for memory for %s"
+                   " (%lld / %d) MiB",
+                   __func__,
+                   action,
+                   phys_mem_available () / 1048576llu,
+                   min_mem);
+        }
+
+      retry_number ++;
+      if (retry_number > retries)
+        return 1;
 
+      sleep (SCHEDULE_PERIOD);
+    }
+  return 0;
+}
 
 /**
  * @brief Perform any syncing that is due.
@@ -5074,7 +5129,11 @@ manage_sync (sigset_t *sigmask_current,
 
   if (feed_sync_required ())
     {
-      if (feed_lockfile_lock (&lockfile) == 0)
+      if (wait_for_mem (check_min_mem_feed_update,
+                        mem_wait_retries,
+                        min_mem_feed_update,
+                        "SecInfo feed sync") == 0
+          && feed_lockfile_lock (&lockfile) == 0)
         {
           pid_t nvts_pid, scap_pid, cert_pid;
           nvts_pid = manage_sync_nvts (fork_update_nvt_cache);
@@ -5096,7 +5155,11 @@ manage_sync (sigset_t *sigmask_current,
           || should_sync_port_lists ()
           || should_sync_report_formats ()))
     {
-      if (feed_lockfile_lock (&lockfile) == 0)
+      if (wait_for_mem (check_min_mem_feed_update,
+                        mem_wait_retries,
+                        min_mem_feed_update,
+                        "data objects feed sync") == 0
+          && feed_lockfile_lock (&lockfile) == 0)
         {
           manage_sync_configs ();
           manage_sync_port_lists ();
@@ -6352,6 +6415,82 @@ set_feed_lock_timeout (int new_timeout)
     feed_lock_timeout = new_timeout;
 }
 
+/**
+ * @brief Get the number of retries when waiting for memory to be available.
+ *
+ * @return The current number of retries.
+ */
+int
+get_mem_wait_retries()
+{
+  return mem_wait_retries;
+}
+
+/**
+ * @brief Set the number of retries when waiting for memory to be available.
+ *
+ * @param[in]  new_retries The new number of retries.
+ */
+void
+set_mem_wait_retries (int new_retries)
+{
+  if (new_retries < 0)
+    min_mem_feed_update = 0;
+  else
+    min_mem_feed_update = new_retries;
+}
+
+/**
+ * @brief Check if the minimum memory for feed updates is available
+ * 
+ * @return 1 if minimum memory amount is available, 0 if not
+ */
+int
+check_min_mem_feed_update ()
+{
+  if (min_mem_feed_update)
+    {
+      guint64 min_mem_bytes = (guint64)min_mem_feed_update * 1048576llu;
+      return phys_mem_available () >= min_mem_bytes ? 1 : 0;
+    }
+  return 1;
+}
+
+/**
+ * @brief Get the minimum memory for feed updates.
+ *
+ * @return The current minimum memory for feed updates in MiB.
+ */
+int
+get_min_mem_feed_update ()
+{
+  return min_mem_feed_update;
+}
+
+/**
+ * @brief Get the minimum memory for feed updates.
+ *
+ * @param[in]  new_min_mem The new minimum memory for feed updates in MiB.
+ */
+void
+set_min_mem_feed_update (int new_min_mem)
+{
+  guint64 min_mem_bytes = (guint64)new_min_mem * 1048576llu;
+  if (new_min_mem < 0)
+    min_mem_feed_update = 0;
+  else if (min_mem_bytes > phys_mem_total ())
+    {
+      g_warning ("%s: requested feed minimum memory limit (%d MiB)"
+                 " exceeds total physical memory (%lld MiB)."
+                 " The setting is ignored.",
+                 __func__,
+                 new_min_mem,
+                 phys_mem_total () / 1048576llu);
+    }
+  else
+    min_mem_feed_update = new_min_mem;
+}
+
 /**
  * @brief Write start time to sync lock file.
  *
diff --git a/src/manage.h b/src/manage.h
index 9e7bbbce9..b60939d56 100644
--- a/src/manage.h
+++ b/src/manage.h
@@ -3870,6 +3870,21 @@ get_feed_lock_timeout ();
 void
 set_feed_lock_timeout (int);
 
+int
+get_mem_wait_retries ();
+
+void
+set_mem_wait_retries (int);
+
+int
+check_min_mem_feed_update ();
+
+int
+get_min_mem_feed_update ();
+
+void
+set_min_mem_feed_update (int);
+
 void
 write_sync_start (int);
 
diff --git a/src/utils.c b/src/utils.c
index 9c7ceb7cc..38455f2c4 100644
--- a/src/utils.c
+++ b/src/utils.c
@@ -959,3 +959,26 @@ wait_for_pid (pid_t pid, const char *context)
         }
     }
 }
+
+/**
+ * @brief Get the available physical memory in bytes.
+ * 
+ * @return The available memory
+ */
+guint64
+phys_mem_available ()
+{
+  return (unsigned long long)(sysconf(_SC_AVPHYS_PAGES))
+    * sysconf(_SC_PAGESIZE);
+}
+
+/**
+ * @brief Get the total physical memory in bytes.
+ * 
+ * @return The total memory
+ */
+guint64
+phys_mem_total ()
+{
+  return sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE);
+}
diff --git a/src/utils.h b/src/utils.h
index 1054dc0ca..8c3e53ff9 100644
--- a/src/utils.h
+++ b/src/utils.h
@@ -103,4 +103,10 @@ fork_with_handlers ();
 void
 wait_for_pid (pid_t, const char *);
 
+guint64
+phys_mem_available ();
+
+guint64
+phys_mem_total ();
+
 #endif /* not _GVMD_UTILS_H */