From fc7b7e172937e6746878fec818a265dd4819c0e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Membr=C3=A9?= Date: Wed, 13 Nov 2019 15:45:00 +0100 Subject: [PATCH 01/16] Fixes #7288: Policy does not get deleted when changing relays --- .../rudder/batch/CleanPromisesFolder.scala | 127 ++++++++++++++++++ .../policies/write/PolicyWriterService.scala | 1 + .../rudder/rest/EndpointsDefinition.scala | 5 + .../rudder/rest/lift/SystemApi.scala | 28 ++++ .../rudder/web/rest/RestTestSetUp.scala | 3 + .../resources/configuration.properties.sample | 16 +++ .../bootstrap/liftweb/RudderConfig.scala | 12 ++ 7 files changed, 192 insertions(+) create mode 100644 webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPromisesFolder.scala diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPromisesFolder.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPromisesFolder.scala new file mode 100644 index 00000000000..030a41b9fff --- /dev/null +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPromisesFolder.scala @@ -0,0 +1,127 @@ +/* +************************************************************************************* +* Copyright 2019 Normation SAS +************************************************************************************* +* +* This file is part of Rudder. +* +* Rudder is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* In accordance with the terms of section 7 (7. Additional Terms.) of +* the GNU General Public License version 3, the copyright holders add +* the following Additional permissions: +* Notwithstanding to the terms of section 5 (5. Conveying Modified Source +* Versions) and 6 (6. Conveying Non-Source Forms.) of the GNU General +* Public License version 3, when you create a Related Module, this +* Related Module is not considered as a part of the work and may be +* distributed under the license agreement of your choice. +* A "Related Module" means a set of sources files including their +* documentation that, without modification of the Source Code, enables +* supplementary functions or services in addition to those offered by +* the Software. +* +* Rudder is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with Rudder. If not, see . + +* +************************************************************************************* +*/ + +package com.normation.rudder.batch + +import better.files.File +import better.files.File.root +import com.normation.inventory.domain.NodeId +import com.normation.rudder.domain.logger.ScheduledJobLogger +import com.normation.rudder.domain.nodes.NodeInfo +import com.normation.rudder.services.nodes.NodeInfoService +import monix.execution.Scheduler.{global => scheduler} +import net.liftweb.common._ +import net.liftweb.util.Helpers.tryo + +import scala.concurrent.duration._ + + +/** + * A naive scheduler which checks every updateInterval if software needs to be deleted + */ +class CleanPoliciesFolder( + nodeInfoService : NodeInfoService + , updateInterval : FiniteDuration +) { + + val logger = ScheduledJobLogger + + + if (updateInterval < 1.hour) { + logger.info(s"Disable automatic unreferenced policies directory (update interval cannot be less than 1 hour)") + } else { + logger.debug(s"***** starting batch that purge unreferenced policies directory, every ${updateInterval.toString()} *****") + scheduler.scheduleWithFixedDelay(1.hour, updateInterval) { + + + } + } + + def launch = { + (for { + nodes <- nodeInfoService.getAll() + deleted <- cleanPoliciesFolderOfDeletedNode(nodes) + } yield { + deleted + }) match { + case eb: EmptyBox => + val error = (eb ?~! s"Error when deleting unreferenced policies directory") + logger.error(error.messageChain) + error.rootExceptionCause.foreach(ex => + logger.error("Exception was:", ex) + ) + error + case Full(deleted) => + logger.info(s"Deleted ${deleted.length} unreferenced policies folder") + if (logger.isDebugEnabled && deleted.length > 0) + logger.debug(s"Deleted policies folder of Nodes: ${deleted.mkString(",")}") + Full(deleted) + } + } + + def cleanPoliciesFolderOfDeletedNode(currentNodes : Map[NodeId,NodeInfo]) : Box[Seq[String]] = { + import File._ + + def getNodeFolders(file: File, parentId: String): Iterator[String] = { + file.children.filter{d => d.name !="rules" && d.isDirectory && (d / "rules").exists}.flatMap { + nodeFolder => + val nodeId = NodeId(nodeFolder.name) + if ( + currentNodes.get(nodeId) match { + case None => false + case Some(nodeInfo) => + val policyServerId = nodeInfo.policyServerId + parentId != policyServerId.value + } + ) { + nodeFolder.delete() + Iterator(nodeFolder.name) + } else { + if ((nodeFolder / "share").exists) { + getNodeFolders(nodeFolder / "share", nodeFolder.name) + } else { + Iterator.empty + } + } + } + } + + tryo(getNodeFolders(root/"var"/"rudder"/"share", "root").toSeq) + } + +} + diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PolicyWriterService.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PolicyWriterService.scala index 26c75e7aa4d..aced97900b4 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PolicyWriterService.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PolicyWriterService.scala @@ -267,6 +267,7 @@ class PolicyWriterServiceImpl( //interpret HookReturnCode as a Box + val readTemplateTime1 = System.currentTimeMillis for { configAndPaths <- calculatePathsForNodeConfigurations(interestingNodeConfigs, rootNodeId, allNodeInfos, newPostfix, backupPostfix) diff --git a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/EndpointsDefinition.scala b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/EndpointsDefinition.scala index 0ddee5e0520..e10840d54cf 100644 --- a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/EndpointsDefinition.scala +++ b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/EndpointsDefinition.scala @@ -402,6 +402,11 @@ object SystemApi extends ApiModuleProvider[SystemApi] { val (action, path) = POST / "system" / "regenerate" / "policies" } + final case object CleanPolicies extends SystemApi with ZeroParam with StartsAtVersion11 with SortIndex { val z = implicitly[Line].value + val description = "Start policies folder cleaning" + val (action, path) = POST / "system" / "clean" / "policies" + } + // Archive list endpoints final case object ArchivesGroupsList extends SystemApi with ZeroParam with StartsAtVersion11 with SortIndex { val z = implicitly[Line].value diff --git a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/SystemApi.scala b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/SystemApi.scala index 1849ca45249..6798276145d 100644 --- a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/SystemApi.scala +++ b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/SystemApi.scala @@ -47,8 +47,10 @@ import com.normation.eventlog.{EventActor, ModificationId} import com.normation.cfclerk.services.{GitRepositoryProvider, UpdateTechniqueLibrary} import com.normation.rudder.batch.{AsyncDeploymentAgent, ManualStartDeployment, UpdateDynamicGroups} import com.normation.rudder.UserService +import com.normation.rudder.batch.CleanPoliciesFolder import com.normation.rudder.repository.xml.{GitFindUtils, GitTagDateTimeFormatter} import com.normation.rudder.repository.{GitArchiveId, GitCommitId, ItemArchiveManager} +import com.normation.rudder.rest.SystemApi.CleanPolicies import com.normation.rudder.services.{ClearCacheService, DebugInfoScriptResult, DebugInfoService} import com.normation.rudder.services.user.PersonIdentService import com.normation.utils.StringUuidGenerator @@ -106,6 +108,7 @@ class SystemApi( case API.GetDirectivesZipArchive => GetDirectivesZipArchive case API.GetRulesZipArchive => GetRulesZipArchive case API.GetAllZipArchive => GetAllZipArchive + case API.CleanPolicies => CleanPolicies }) } @@ -149,6 +152,15 @@ class SystemApi( } + object CleanPolicies extends LiftApiModule0 { + val schema = API.CleanPolicies + val restExtractor = restExtractorService + + def process0(version: ApiVersion, path: ApiPath, req: Req, params: DefaultParams, authzToken: AuthzToken): LiftResponse = { + apiv11service.cleanPolicies(req, params) + } + } + //Reload [techniques / dynamics groups] endpoint implementation object TechniquesReload extends LiftApiModule0 { @@ -428,6 +440,7 @@ class SystemApiService11( , itemArchiveManager : ItemArchiveManager , personIdentService : PersonIdentService , repo : GitRepositoryProvider + , cleanPoliciesFolder : CleanPoliciesFolder ) (implicit userService: UserService) extends Loggable { @@ -937,4 +950,19 @@ class SystemApiService11( asyncDeploymentAgent.launchDeployment(dest) toJsonResponse(None, "policies" -> "Started") } + + + def cleanPolicies(req: Req, params: DefaultParams) : LiftResponse = { + + implicit val action = "cleanPolicies" + implicit val prettify = params.prettify + + cleanPoliciesFolder.launch match { + case emptyBox: EmptyBox => + val fail = emptyBox ?~! "An error while cleaning policies folder through rest API" + toJsonError(None, fail.messageChain) + case Full(deleted) => + toJsonResponse(None, ("length" -> deleted.length) ~ ("nodes" -> deleted)) + } + } } diff --git a/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/web/rest/RestTestSetUp.scala b/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/web/rest/RestTestSetUp.scala index c42596495af..8de1d32e702 100644 --- a/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/web/rest/RestTestSetUp.scala +++ b/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/web/rest/RestTestSetUp.scala @@ -59,6 +59,7 @@ import com.normation.rudder.User import com.normation.rudder.AuthorizationType import com.normation.rudder.RudderAccount import com.normation.rudder.api.{ApiAuthorization => ApiAuthz} +import com.normation.rudder.batch.CleanPoliciesFolder import com.normation.rudder.batch.{AsyncDeploymentAgent, StartDeploymentMessage, UpdateDynamicGroups} import com.normation.rudder.repository._ import com.normation.rudder.rest.v1.RestTechniqueReload @@ -69,6 +70,7 @@ import com.normation.rudder.services.policies.TestNodeConfiguration import com.normation.rudder.services.user.PersonIdentService import org.eclipse.jgit.lib.PersonIdent import org.joda.time.DateTime +import scala.concurrent.duration._ /* @@ -217,6 +219,7 @@ object RestTestSetUp { , fakeItemArchiveManager , fakePersonIndentService , fakeRepo + , new CleanPoliciesFolder(???, 24.hours) ) val systemApi = new com.normation.rudder.rest.lift.SystemApi(restExtractorService, apiService11, "5.0", "5.0.0", "some time") diff --git a/webapp/sources/rudder/rudder-web/src/main/resources/configuration.properties.sample b/webapp/sources/rudder/rudder-web/src/main/resources/configuration.properties.sample index 30a9e1c83de..96b6a033d5c 100644 --- a/webapp/sources/rudder/rudder-web/src/main/resources/configuration.properties.sample +++ b/webapp/sources/rudder/rudder-web/src/main/resources/configuration.properties.sample @@ -228,6 +228,22 @@ rudder.batch.purge.inventories.delete.TTL=7 # Interval in hours rudder.batch.purge.inventories.delete.interval=24 +########################### +# Promise folder cleaning ########################################################### +########################### + +# +# Promise folder cleaning +# This allows you to schedule the purge of unused promised folders (Deleted Nodes, or Nodes that has moved to another relay/policy server) +# +# +# Defaults: check runs every 24 hours. +# + +# Interval in hours +rudder.batch.clean.promises.interval=24 + + ##################### # Webdav properties ################################################################# ##################### diff --git a/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala b/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala index fc0b0ae87da..c486a73c04d 100644 --- a/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala +++ b/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala @@ -352,6 +352,16 @@ object RudderConfig extends Loggable { } } + val RUDDER_BATCH_CLEAN_PROMISES_INTERVAL = { + try { + config.getInt("rudder.batch.clean.policies.interval") + } catch { + case ex: ConfigException => + ApplicationLogger.info("Property 'rudder.batch.clean.policies.interval' is missing or empty in rudder.configFile. Default to 24 hours.") + 24 + } + } + // Roles definitions val RUDDER_SERVER_ROLES = Seq( //each time, it's (role name, key in the config file) @@ -462,6 +472,7 @@ object RudderConfig extends Loggable { val checkInventoryUpdate = new CheckInventoryUpdate(nodeInfoServiceImpl, asyncDeploymentAgent, stringUuidGenerator, 15.seconds) val purgeDeletedInventories = new PurgeDeletedInventories(removeNodeServiceImpl, RUDDER_BATCH_PURGE_DELETED_INVENTORIES_INTERVAL.hours, RUDDER_BATCH_PURGE_DELETED_INVENTORIES) val purgeUnreferencedSoftwares = new PurgeUnreferencedSoftwares(softwareService, RUDDER_BATCH_DELETE_SOFTWARE_INTERVAL.hours) + val cleanPoliciesFolder = new CleanPoliciesFolder(nodeInfoServiceImpl, RUDDER_BATCH_CLEAN_PROMISES_INTERVAL.hours) val databaseManager: DatabaseManager = databaseManagerImpl val automaticReportsCleaning: AutomaticReportsCleaning = dbCleaner val checkTechniqueLibrary: CheckTechniqueLibrary = techniqueLibraryUpdater @@ -769,6 +780,7 @@ object RudderConfig extends Loggable { , itemArchiveManager , personIdentService , gitRepo + , cleanPoliciesFolder ) private[this] val complianceAPIService = new ComplianceAPIService( From 578a3731d4311db3dfa0e906da50ffbd96bcc964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Membr=C3=A9?= Date: Thu, 21 Nov 2019 14:15:27 +0100 Subject: [PATCH 02/16] fixup! Fixes #7288: Policy does not get deleted when changing relays Fixes #7288: Policy does not get deleted when changing relays --- .../{CleanPromisesFolder.scala => CleanPoliciesFolder.scala} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/{CleanPromisesFolder.scala => CleanPoliciesFolder.scala} (99%) diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPromisesFolder.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala similarity index 99% rename from webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPromisesFolder.scala rename to webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala index 030a41b9fff..317006de00a 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPromisesFolder.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala @@ -102,7 +102,7 @@ class CleanPoliciesFolder( val nodeId = NodeId(nodeFolder.name) if ( currentNodes.get(nodeId) match { - case None => false + case None => true case Some(nodeInfo) => val policyServerId = nodeInfo.policyServerId parentId != policyServerId.value From bd706926ce729c662f48a52ba6728e09db60a8fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Membr=C3=A9?= Date: Thu, 21 Nov 2019 15:28:50 +0100 Subject: [PATCH 03/16] fixup! fixup! Fixes #7288: Policy does not get deleted when changing relays Fixes #7288: Policy does not get deleted when changing relays --- .../com/normation/rudder/batch/CleanPoliciesFolder.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala index 317006de00a..33b595b5548 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala @@ -45,9 +45,10 @@ import com.normation.rudder.domain.nodes.NodeInfo import com.normation.rudder.services.nodes.NodeInfoService import monix.execution.Scheduler.{global => scheduler} import net.liftweb.common._ -import net.liftweb.util.Helpers.tryo import scala.concurrent.duration._ +import scala.util.{Success, Failure => Catch} +import scala.util.Try /** @@ -120,7 +121,10 @@ class CleanPoliciesFolder( } } - tryo(getNodeFolders(root/"var"/"rudder"/"share", "root").toSeq) + Try(getNodeFolders(root/"var"/"rudder"/"share", "root").toSeq) match { + case Success(value) => Full(value) + case Catch(e) => Failure(e.getMessage, Full(e), Empty) + } } } From 2349b88366545f6bd88d37b097094fb7d4830ff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Membr=C3=A9?= Date: Thu, 21 Nov 2019 16:00:45 +0100 Subject: [PATCH 04/16] fixup! fixup! fixup! Fixes #7288: Policy does not get deleted when changing relays Fixes #7288: Policy does not get deleted when changing relays --- .../com/normation/rudder/batch/CleanPoliciesFolder.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala index 33b595b5548..75d6ec36fb3 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala @@ -75,7 +75,8 @@ class CleanPoliciesFolder( def launch = { (for { nodes <- nodeInfoService.getAll() - deleted <- cleanPoliciesFolderOfDeletedNode(nodes) + d <- Box.tryo{cleanPoliciesFolderOfDeletedNode(nodes)} + deleted <- d } yield { deleted }) match { @@ -98,7 +99,7 @@ class CleanPoliciesFolder( import File._ def getNodeFolders(file: File, parentId: String): Iterator[String] = { - file.children.filter{d => d.name !="rules" && d.isDirectory && (d / "rules").exists}.flatMap { + file.children.filter{d => d.name !="rules" && d.isDirectory && (d / "rules").exists}.map(_.autoClosed).flatMap { nodeFolder => val nodeId = NodeId(nodeFolder.name) if ( @@ -110,7 +111,9 @@ class CleanPoliciesFolder( } ) { nodeFolder.delete() - Iterator(nodeFolder.name) + val res = Iterator(nodeFolder.name) + nodeFolder.autoClosed + res } else { if ((nodeFolder / "share").exists) { getNodeFolders(nodeFolder / "share", nodeFolder.name) From 2e8ebc7113cf3a9ee15c727b7816eddcd3b38082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Membr=C3=A9?= Date: Thu, 21 Nov 2019 16:03:05 +0100 Subject: [PATCH 05/16] fixup! fixup! fixup! fixup! Fixes #7288: Policy does not get deleted when changing relays Fixes #7288: Policy does not get deleted when changing relays --- .../scala/com/normation/rudder/batch/CleanPoliciesFolder.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala index 75d6ec36fb3..ca959291203 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala @@ -99,7 +99,7 @@ class CleanPoliciesFolder( import File._ def getNodeFolders(file: File, parentId: String): Iterator[String] = { - file.children.filter{d => d.name !="rules" && d.isDirectory && (d / "rules").exists}.map(_.autoClosed).flatMap { + file.children.filter{d => d.name !="rules" && d.isDirectory && (d / "rules").exists}.flatMap { nodeFolder => val nodeId = NodeId(nodeFolder.name) if ( @@ -112,7 +112,6 @@ class CleanPoliciesFolder( ) { nodeFolder.delete() val res = Iterator(nodeFolder.name) - nodeFolder.autoClosed res } else { if ((nodeFolder / "share").exists) { From 3277ceb5d7a072c992b684a62eb026c21b550244 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Membr=C3=A9?= Date: Thu, 21 Nov 2019 16:34:45 +0100 Subject: [PATCH 06/16] fixup! fixup! fixup! fixup! fixup! Fixes #7288: Policy does not get deleted when changing relays Fixes #7288: Policy does not get deleted when changing relays --- .../rudder/batch/CleanPoliciesFolder.scala | 45 ++++++++++++------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala index ca959291203..a64e55eb93e 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala @@ -99,29 +99,40 @@ class CleanPoliciesFolder( import File._ def getNodeFolders(file: File, parentId: String): Iterator[String] = { - file.children.filter{d => d.name !="rules" && d.isDirectory && (d / "rules").exists}.flatMap { + file.children.flatMap { nodeFolder => - val nodeId = NodeId(nodeFolder.name) - if ( - currentNodes.get(nodeId) match { - case None => true - case Some(nodeInfo) => - val policyServerId = nodeInfo.policyServerId - parentId != policyServerId.value - } - ) { - nodeFolder.delete() - val res = Iterator(nodeFolder.name) - res - } else { - if ((nodeFolder / "share").exists) { - getNodeFolders(nodeFolder / "share", nodeFolder.name) + val folderName = nodeFolder.name + if (folderName !="rules") { + if (nodeFolder.isDirectory && (nodeFolder / "rules").exists) { + val nodeId = NodeId(nodeFolder.name) + if ( + currentNodes.get(nodeId) match { + case None => true + case Some(nodeInfo) => + val policyServerId = nodeInfo.policyServerId + parentId != policyServerId.value + } + ) { + nodeFolder.delete() + val res = Iterator(nodeFolder.name) + res + } else { + if ((nodeFolder / "share").exists) { + getNodeFolders(nodeFolder / "share", nodeFolder.name) + } else { + Iterator.empty + } + } } else { Iterator.empty } + } + else { + Iterator.empty + } } } - } + Try(getNodeFolders(root/"var"/"rudder"/"share", "root").toSeq) match { case Success(value) => Full(value) From cf1930f5dd7d66c60fd9dac97379f58ecb910813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Membr=C3=A9?= Date: Thu, 21 Nov 2019 17:40:12 +0100 Subject: [PATCH 07/16] fixup! fixup! fixup! fixup! fixup! fixup! Fixes #7288: Policy does not get deleted when changing relays Fixes #7288: Policy does not get deleted when changing relays --- .../com/normation/rudder/batch/CleanPoliciesFolder.scala | 6 +++--- .../scala/com/normation/rudder/rest/lift/SystemApi.scala | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala index a64e55eb93e..421eb6a5a33 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala @@ -95,7 +95,7 @@ class CleanPoliciesFolder( } } - def cleanPoliciesFolderOfDeletedNode(currentNodes : Map[NodeId,NodeInfo]) : Box[Seq[String]] = { + def cleanPoliciesFolderOfDeletedNode(currentNodes : Map[NodeId,NodeInfo]) : Box[List[String]] = { import File._ def getNodeFolders(file: File, parentId: String): Iterator[String] = { @@ -132,9 +132,9 @@ class CleanPoliciesFolder( } } } - - Try(getNodeFolders(root/"var"/"rudder"/"share", "root").toSeq) match { + + Try(getNodeFolders(root/"var"/"rudder"/"share", "root").toList) match { case Success(value) => Full(value) case Catch(e) => Failure(e.getMessage, Full(e), Empty) } diff --git a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/SystemApi.scala b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/SystemApi.scala index 6798276145d..a52fe5fa1cb 100644 --- a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/SystemApi.scala +++ b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/SystemApi.scala @@ -50,7 +50,6 @@ import com.normation.rudder.UserService import com.normation.rudder.batch.CleanPoliciesFolder import com.normation.rudder.repository.xml.{GitFindUtils, GitTagDateTimeFormatter} import com.normation.rudder.repository.{GitArchiveId, GitCommitId, ItemArchiveManager} -import com.normation.rudder.rest.SystemApi.CleanPolicies import com.normation.rudder.services.{ClearCacheService, DebugInfoScriptResult, DebugInfoService} import com.normation.rudder.services.user.PersonIdentService import com.normation.utils.StringUuidGenerator From a27e0253d4a0d555604d8b143792d8ee49e05f77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Membr=C3=A9?= Date: Thu, 21 Nov 2019 17:57:39 +0100 Subject: [PATCH 08/16] fixup! fixup! fixup! fixup! fixup! fixup! fixup! Fixes #7288: Policy does not get deleted when changing relays Fixes #7288: Policy does not get deleted when changing relays --- .../rudder/batch/CleanPoliciesFolder.scala | 42 +++++++------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala index 421eb6a5a33..24d821337c0 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala @@ -101,37 +101,27 @@ class CleanPoliciesFolder( def getNodeFolders(file: File, parentId: String): Iterator[String] = { file.children.flatMap { nodeFolder => - val folderName = nodeFolder.name - if (folderName !="rules") { - if (nodeFolder.isDirectory && (nodeFolder / "rules").exists) { - val nodeId = NodeId(nodeFolder.name) - if ( - currentNodes.get(nodeId) match { - case None => true - case Some(nodeInfo) => - val policyServerId = nodeInfo.policyServerId - parentId != policyServerId.value - } - ) { - nodeFolder.delete() - val res = Iterator(nodeFolder.name) - res - } else { - if ((nodeFolder / "share").exists) { - getNodeFolders(nodeFolder / "share", nodeFolder.name) - } else { - Iterator.empty - } - } + val nodeId = NodeId(nodeFolder.name) + if ( + currentNodes.get(nodeId) match { + case None => true + case Some(nodeInfo) => + val policyServerId = nodeInfo.policyServerId + parentId != policyServerId.value + } + ) { + nodeFolder.delete() + val res = Iterator(nodeFolder.name) + res + } else { + if ((nodeFolder / "share").exists) { + getNodeFolders(nodeFolder / "share", nodeFolder.name) } else { Iterator.empty } - } - else { - Iterator.empty - } } } + } Try(getNodeFolders(root/"var"/"rudder"/"share", "root").toList) match { From 4f8e8f2462e2c854f8965fe5ba3df0c57e8bfacd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Membr=C3=A9?= Date: Fri, 22 Nov 2019 10:46:57 +0100 Subject: [PATCH 09/16] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Fixes #7288: Policy does not get deleted when changing relays Fixes #7288: Policy does not get deleted when changing relays --- .../rudder/batch/CleanPoliciesFolder.scala | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala index 24d821337c0..e470b3e2711 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala @@ -37,6 +37,8 @@ package com.normation.rudder.batch +import java.nio.file.FileSystemException + import better.files.File import better.files.File.root import com.normation.inventory.domain.NodeId @@ -124,9 +126,14 @@ class CleanPoliciesFolder( } - Try(getNodeFolders(root/"var"/"rudder"/"share", "root").toList) match { - case Success(value) => Full(value) - case Catch(e) => Failure(e.getMessage, Full(e), Empty) + try { + Try(getNodeFolders(root / "var" / "rudder" / "share", "root").toList) match { + case Success(value) => Full(value) + case Catch(e) => Failure(e.getMessage, Full(e), Empty) + } + } catch { + case fse : FileSystemException => + Failure(fse.getMessage, Full(fse), Empty) } } From 6bceb27fa20f22900b7b6ce5c45aa64527df7ba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Membr=C3=A9?= Date: Fri, 22 Nov 2019 11:11:01 +0100 Subject: [PATCH 10/16] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Fixes #7288: Policy does not get deleted when changing relays Fixes #7288: Policy does not get deleted when changing relays --- .../com/normation/rudder/batch/CleanPoliciesFolder.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala index e470b3e2711..cf2237ea11d 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/CleanPoliciesFolder.scala @@ -86,7 +86,7 @@ class CleanPoliciesFolder( val error = (eb ?~! s"Error when deleting unreferenced policies directory") logger.error(error.messageChain) error.rootExceptionCause.foreach(ex => - logger.error("Exception was:", ex) + logger.debug("Exception was:", ex) ) error case Full(deleted) => @@ -133,7 +133,7 @@ class CleanPoliciesFolder( } } catch { case fse : FileSystemException => - Failure(fse.getMessage, Full(fse), Empty) + Failure("Too many files open, maybe you can augment fd limit (ulimit)", Full(fse), Empty) } } From e56e922aa0e16bdf34965fa4d62a3458909ccf8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Membr=C3=A9?= Date: Fri, 22 Nov 2019 11:35:16 +0100 Subject: [PATCH 11/16] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Fixes #7288: Policy does not get deleted when changing relays Fixes #7288: Policy does not get deleted when changing relays --- webapp/sources/rudder-parent-pom/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/sources/rudder-parent-pom/pom.xml b/webapp/sources/rudder-parent-pom/pom.xml index 73940970854..84803ea62bf 100644 --- a/webapp/sources/rudder-parent-pom/pom.xml +++ b/webapp/sources/rudder-parent-pom/pom.xml @@ -382,7 +382,7 @@ limitations under the License. 1.60 2.3.3 1.2 - 3.7.0 + 3.8.0