From 1261254a618fdf2a4a7dd2ab7f44ff0eeeab427c Mon Sep 17 00:00:00 2001
From: Yonas Habteab <yonas.habteab@icinga.com>
Date: Mon, 16 Sep 2024 09:20:43 +0200
Subject: [PATCH] ApiListener: Sync runtime configs in order

---
 lib/remote/apilistener-configsync.cpp | 73 ++++++++++++++++++++++-----
 lib/remote/apilistener.hpp            |  2 +
 2 files changed, 61 insertions(+), 14 deletions(-)

diff --git a/lib/remote/apilistener-configsync.cpp b/lib/remote/apilistener-configsync.cpp
index 04436ad8b99..5f0a622fc26 100644
--- a/lib/remote/apilistener-configsync.cpp
+++ b/lib/remote/apilistener-configsync.cpp
@@ -5,11 +5,13 @@
 #include "remote/configobjectutility.hpp"
 #include "remote/jsonrpc.hpp"
 #include "base/configtype.hpp"
-#include "base/json.hpp"
 #include "base/convert.hpp"
+#include "base/dependencygraph.hpp"
+#include "base/json.hpp"
 #include "config/vmops.hpp"
 #include "remote/configobjectslock.hpp"
 #include <fstream>
+#include <unordered_set>
 
 using namespace icinga;
 
@@ -393,6 +395,51 @@ void ApiListener::UpdateConfigObject(const ConfigObject::Ptr& object, const Mess
 	}
 }
 
+/**
+ * Syncs the specified object and its direct and indirect parents to the provided client
+ * in topological order of their dependency graph recursively.
+ *
+ * Objects that the client does not have access to are skipped without going through their dependency graph.
+ *
+ * Please do not use this method to forward remote generated cluster updates; it should only be used to
+ * send local updates to that specific non-nullptr client.
+ *
+ * @param object The config object you want to sync.
+ * @param client The JsonRpc client you send the update to.
+ * @param syncedObjects Used to cache the already synced objects.
+ */
+void ApiListener::UpdateConfigObjectWithParents(const ConfigObject::Ptr& object, const JsonRpcConnection::Ptr& client,
+	std::unordered_set<ConfigObject*>& syncedObjects)
+{
+	if (syncedObjects.find(object.get()) != syncedObjects.end()) {
+		return;
+	}
+
+	VERIFY(client);
+
+	Endpoint::Ptr endpoint = client->GetEndpoint();
+	ASSERT(endpoint);
+
+	Zone::Ptr azone = endpoint->GetZone();
+
+	/* don't sync objects for non-matching parent-child zones */
+	if (!azone->CanAccessObject(object)) {
+		return;
+	}
+	syncedObjects.emplace(object.get());
+
+	for (const auto& parent : DependencyGraph::GetParents(object)) {
+		// Actually, the following dynamic cast should never fail, since the DependencyGraph class
+		// expects the types to always be of type Object::Ptr and such an object is supposed to always
+		// point to an instance of the specific derived ConfigObject class. See TypeHelper<>::GetFactory().
+		if (auto parentObj = dynamic_pointer_cast<ConfigObject>(parent)) {
+			UpdateConfigObjectWithParents(parentObj, client, syncedObjects);
+		}
+	}
+
+	/* send the config object to the connected client */
+	UpdateConfigObject(object, nullptr, client);
+}
 
 void ApiListener::DeleteConfigObject(const ConfigObject::Ptr& object, const MessageOrigin::Ptr& origin,
 	const JsonRpcConnection::Ptr& client)
@@ -454,19 +501,17 @@ void ApiListener::SendRuntimeConfigObjects(const JsonRpcConnection::Ptr& aclient
 	Log(LogInformation, "ApiListener")
 		<< "Syncing runtime objects to endpoint '" << endpoint->GetName() << "'.";
 
-	for (const Type::Ptr& type : Type::GetAllTypes()) {
-		auto *dtype = dynamic_cast<ConfigType *>(type.get());
-
-		if (!dtype)
-			continue;
-
-		for (const ConfigObject::Ptr& object : dtype->GetObjects()) {
-			/* don't sync objects for non-matching parent-child zones */
-			if (!azone->CanAccessObject(object))
-				continue;
-
-			/* send the config object to the connected client */
-			UpdateConfigObject(object, nullptr, aclient);
+	std::unordered_set<ConfigObject*> syncedObjects;
+	for (const auto& type : Type::GetConfigTypesSortedByLoadDependencies()) {
+		if (auto *ctype = dynamic_cast<ConfigType *>(type.get())) {
+			for (const auto& object : ctype->GetObjects()) {
+				// All objects must be synced sorted by their dependency graph.
+				// Otherwise, downtimes/comments etc. might get synced before their respective Checkables, which will
+				// result in comments and downtimes being ignored by the other endpoint since it does not yet know
+				// about their checkables. Given that the runtime config updates event does not trigger a reload on the
+				// remote endpoint, these objects won't be synced again until the next reload.
+				UpdateConfigObjectWithParents(object, aclient, syncedObjects);
+			}
 		}
 	}
 
diff --git a/lib/remote/apilistener.hpp b/lib/remote/apilistener.hpp
index fced0a8afb1..e047f643346 100644
--- a/lib/remote/apilistener.hpp
+++ b/lib/remote/apilistener.hpp
@@ -247,6 +247,8 @@ class ApiListener final : public ObjectImpl<ApiListener>
 	/* configsync */
 	void UpdateConfigObject(const ConfigObject::Ptr& object, const MessageOrigin::Ptr& origin,
 		const JsonRpcConnection::Ptr& client = nullptr);
+	void UpdateConfigObjectWithParents(const ConfigObject::Ptr& object, const JsonRpcConnection::Ptr& client,
+		std::unordered_set<ConfigObject*>& syncedObjects);
 	void DeleteConfigObject(const ConfigObject::Ptr& object, const MessageOrigin::Ptr& origin,
 		const JsonRpcConnection::Ptr& client = nullptr);
 	void SendRuntimeConfigObjects(const JsonRpcConnection::Ptr& aclient);