Skipping internal setup of ES and PG in case of External config (#6606)

* Changes for External PG Signed-off-by: shanmugapriya-tr <[email protected]> * Refactored skipping services health check Signed-off-by: shanmugapriya-tr <[email protected]> * Changes for External ES 1. Added the code to skip the automate-elasticsearch service if External is enabled 2. Decoupled the es-sidecar service from automate-elasticsearch 3. Moved the code to skip the status check for removed services in ensureStatus Signed-off-by: sandhi <[email protected]> * Added few debug logs * Added few debug logs Signed-off-by: shanmugapriya-tr <[email protected]> * Making binds optional Signed-off-by: shanmugapriya-tr <[email protected]> * Changes for External PG Signed-off-by: shanmugapriya-tr <[email protected]> * Refactored skipping services health check Signed-off-by: shanmugapriya-tr <[email protected]> * Changes for External ES 1. Added the code to skip the automate-elasticsearch service if External is enabled 2. Decoupled the es-sidecar service from automate-elasticsearch 3. Moved the code to skip the status check for removed services in ensureStatus Signed-off-by: sandhi <[email protected]> * Added few debug logs * Making binds optional Signed-off-by: shanmugapriya-tr <[email protected]> * Changed Service names to constants Signed-off-by: shanmugapriya-tr <[email protected]> * Refactored code Signed-off-by: shanmugapriya-tr <[email protected]> * Changes to resolve the issue with service restart on patch Signed-off-by: sandhi <[email protected]> * Update components/automate-deployment/pkg/server/server.go Co-authored-by: Kallol Roy <[email protected]> * Changed array to map Signed-off-by: shanmugapriya-tr <[email protected]> * Changes for External PG Signed-off-by: shanmugapriya-tr <[email protected]> * Refactored skipping services health check Signed-off-by: shanmugapriya-tr <[email protected]> * Changes for External ES 1. Added the code to skip the automate-elasticsearch service if External is enabled 2. Decoupled the es-sidecar service from automate-elasticsearch 3. Moved the code to skip the status check for removed services in ensureStatus Signed-off-by: sandhi <[email protected]> * Added few debug logs * Making binds optional Signed-off-by: shanmugapriya-tr <[email protected]> * Changes for External PG Signed-off-by: shanmugapriya-tr <[email protected]> * Changed Service names to constants Signed-off-by: shanmugapriya-tr <[email protected]> * Refactored code Signed-off-by: shanmugapriya-tr <[email protected]> * Changes to resolve the issue with service restart on patch Signed-off-by: sandhi <[email protected]> * Update components/automate-deployment/pkg/server/server.go Co-authored-by: Kallol Roy <[email protected]> * Changed array to map Signed-off-by: shanmugapriya-tr <[email protected]> * Changes for External PG Signed-off-by: shanmugapriya-tr <[email protected]> * Refactored skipping services health check Signed-off-by: shanmugapriya-tr <[email protected]> * Changes for External ES 1. Added the code to skip the automate-elasticsearch service if External is enabled 2. Decoupled the es-sidecar service from automate-elasticsearch 3. Moved the code to skip the status check for removed services in ensureStatus Signed-off-by: sandhi <[email protected]> * Added few debug logs * Making binds optional Signed-off-by: shanmugapriya-tr <[email protected]> * Changed Service names to constants Signed-off-by: shanmugapriya-tr <[email protected]> * Refactored code Signed-off-by: shanmugapriya-tr <[email protected]> * Changes to resolve the issue with service restart on patch Signed-off-by: sandhi <[email protected]> * Update components/automate-deployment/pkg/server/server.go Co-authored-by: Kallol Roy <[email protected]> * Changed array to map Signed-off-by: shanmugapriya-tr <[email protected]> * Changes for External PG Signed-off-by: shanmugapriya-tr <[email protected]> * added automateHA mount path (#6739) * added automateHA mount path Signed-off-by: ArvinthC3000 <[email protected]> * added automateHA mount path - dynamic path test Signed-off-by: ArvinthC3000 <[email protected]> * Added backup_mount variable * backup_mount path from variables_common added(test commit) Signed-off-by: ArvinthC3000 <[email protected]> * reverting changes Signed-off-by: ArvinthC3000 <[email protected]> * Adding dynamic mount path to automate backup folder creation Signed-off-by: ArvinthC3000 <[email protected]> * Static backup mount path * trigger changes Signed-off-by: ArvinthC3000 <[email protected]> * removing dead code Signed-off-by: ArvinthC3000 <[email protected]> * Rename HA_architecture_reference.md to ha_architecture_reference.md * removed unused config keys (#6781) Signed-off-by: ArvinthC3000 <[email protected]> * Changes for External PG Signed-off-by: shanmugapriya-tr <[email protected]> * Refactored skipping services health check Signed-off-by: shanmugapriya-tr <[email protected]> * Changes for External ES 1. Added the code to skip the automate-elasticsearch service if External is enabled 2. Decoupled the es-sidecar service from automate-elasticsearch 3. Moved the code to skip the status check for removed services in ensureStatus Signed-off-by: sandhi <[email protected]> * Added few debug logs * Making binds optional Signed-off-by: shanmugapriya-tr <[email protected]> * Changed Service names to constants Signed-off-by: shanmugapriya-tr <[email protected]> * Refactored code Signed-off-by: shanmugapriya-tr <[email protected]> * Changes to resolve the issue with service restart on patch Signed-off-by: sandhi <[email protected]> * Update components/automate-deployment/pkg/server/server.go Co-authored-by: Kallol Roy <[email protected]> * Changed array to map Signed-off-by: shanmugapriya-tr <[email protected]> * Changes for External PG Signed-off-by: shanmugapriya-tr <[email protected]> * Fixing License Scout (#6782) * Fixing License Scout Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * Fixing License Scout for msgp Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * Cli milestone part-1 (#6762) * added Todo(milestone) for the servers Signed-off-by: Kallol Roy <[email protected]> * added more Todo(milestone) for the servers Signed-off-by: Kallol Roy <[email protected]> * adding to-dos Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * upgrading to minor/patch version (#6621) * upgrade to minor\patch Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * supporting sem version Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * considering version flag in upgrade Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * compute min current version for airgap bundle create- second pr (#6665) * adding min current version to manifest in bundle creation Signed-off-by: Rick Marry <[email protected]> * adding min current version to manifest in bundle creation Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> Co-authored-by: dmaddu <[email protected]> * major upgrade with checklist (#6660) Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * fixing buildkite (#6674) Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * airgap bundle upgrade (#6682) * airgap upgrade Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * upgrade status for airgap Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * upgrade status for airgap instance Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * addressing review comments Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * Show Major Version Release Banner on UI (#6663) * banner-manual-upgrade changes Signed-off-by: Venkatesh-rengasamy <[email protected]> * add banner logo Signed-off-by: Venkatesh-rengasamy <[email protected]> * updating error messages to be shown to end user (#6714) Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * ignoring semgrep rule Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * fixing pipeline issue (#6717) Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * supporting external versions file (#6748) * supporting external versions file Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * fixing pipeline Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * showing pre post Checklist for milestone release (#6706) * postchecklist Signed-off-by: shaik80 <[email protected]> * json file Signed-off-by: shaik80 <[email protected]> * readPostCheckList Signed-off-by: shaik80 <[email protected]> * read json file for postchecklist Signed-off-by: shaik80 <[email protected]> * upgrade to major and and print postchecklist steps Signed-off-by: shaik80 <[email protected]> * test cases and basic changes Signed-off-by: shaik80 <[email protected]> * dynamic version Signed-off-by: shaik80 <[email protected]> * old manifests url Signed-off-by: shaik80 <[email protected]> * error handling in upgrade status if we don't hav compatible version Signed-off-by: shaik80 <[email protected]> * update json file func changes Signed-off-by: shaik80 <[email protected]> * add version in json file Signed-off-by: shaik80 <[email protected]> * Pre and post checklist for embedded and external PG (#6705) * changes for pre post checklist Signed-off-by: Vivek Shankar <[email protected]> * minor changes Signed-off-by: Vivek Shankar <[email protected]> * changed formatting Signed-off-by: Vivek Shankar <[email protected]> * id changes Signed-off-by: Vivek Shankar <[email protected]> * Pg external Signed-off-by: shaik80 <[email protected]> * minor fmt changes Signed-off-by: Vivek Shankar <[email protected]> * spell check Signed-off-by: shaik80 <[email protected]> * variable change Signed-off-by: shaik80 <[email protected]> * remove log Signed-off-by: shaik80 <[email protected]> * postchecklist new structure Signed-off-by: shaik80 <[email protected]> * added new interface function for post checklist Signed-off-by: Vivek Yadav <[email protected]> * get post checklist implementation added Signed-off-by: Vivek Yadav <[email protected]> * feed back changes Signed-off-by: shaik80 <[email protected]> * code modification Signed-off-by: shaik80 <[email protected]> * if file not found return nil Signed-off-by: shaik80 <[email protected]> * if file not found don't throw an error Signed-off-by: shaik80 <[email protected]> * remove log and lint fix Signed-off-by: shaik80 <[email protected]> * NewChecklistManager remove major version param Signed-off-by: shaik80 <[email protected]> * Fixing pipeline issues Signed-off-by: shaik80 <[email protected]> * title for post checklist in upgrade status Signed-off-by: shaik80 <[email protected]> * making global constant Signed-off-by: shaik80 <[email protected]> Co-authored-by: vivekshankar1 <[email protected]> Co-authored-by: Vivek Shankar <[email protected]> Co-authored-by: Vivek Yadav <[email protected]> Co-authored-by: dmaddu <[email protected]> * partial changes to fix pipeline Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * hiding versions-file flag to end users Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * removing the commented code Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * hiding the banner Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * Fix automate UI pipeline issue (#6790) * pipline fix Signed-off-by: shaik80 <[email protected]> * automate pipline fix Signed-off-by: shaik80 <[email protected]> Co-authored-by: shaik80 <[email protected]> Co-authored-by: Kallol Roy <[email protected]> Co-authored-by: Rick Marry <[email protected]> Co-authored-by: Venkatesh-rengasamy <[email protected]> Co-authored-by: Shaik Mudassir <[email protected]> Co-authored-by: vivekshankar1 <[email protected]> Co-authored-by: Vivek Shankar <[email protected]> Co-authored-by: Vivek Yadav <[email protected]> Co-authored-by: shivangisingh <[email protected]> Co-authored-by: shaik80 <[email protected]> * Executed '.expeditor/generate-automate-cli-docs.sh' Obvious fix; these changes are the result of automation not creative thinking. * regenerating the cert files (#6794) Signed-off-by: Durga Sarat Chandra Maddu <[email protected]> * Upgrade automate ha packages for 6.8.6 -> 6.8.23 (#6795) * upgrade es version to 6.8.23 in automate-ha * side car package bump to avaoid the confusion Signed-off-by: punitmundra <[email protected]> * License scout fix (#6798) * set git config global Signed-off-by: Vivek Yadav <[email protected]> * updated commit id of epgsql Signed-off-by: Vivek Yadav <[email protected]> * log git config updated in license scout Signed-off-by: Vivek Yadav <[email protected]> * changed url in git config override Signed-off-by: Vivek Yadav <[email protected]> * fix git:// in build and unit test of notification Signed-off-by: Vivek Yadav <[email protected]> * Changes for External PG Signed-off-by: shanmugapriya-tr <[email protected]> * Refactored skipping services health check Signed-off-by: shanmugapriya-tr <[email protected]> * Changes for External ES 1. Added the code to skip the automate-elasticsearch service if External is enabled 2. Decoupled the es-sidecar service from automate-elasticsearch 3. Moved the code to skip the status check for removed services in ensureStatus Signed-off-by: sandhi <[email protected]> * Added few debug logs * Making binds optional Signed-off-by: shanmugapriya-tr <[email protected]> * Changed Service names to constants Signed-off-by: shanmugapriya-tr <[email protected]> * Refactored code Signed-off-by: shanmugapriya-tr <[email protected]> * Changes to resolve the issue with service restart on patch Signed-off-by: sandhi <[email protected]> * Update components/automate-deployment/pkg/server/server.go Co-authored-by: Kallol Roy <[email protected]> * Changed array to map Signed-off-by: shanmugapriya-tr <[email protected]> * Changes for External PG Signed-off-by: shanmugapriya-tr <[email protected]> Co-authored-by: shanmugapriya-tr <[email protected]> Co-authored-by: sandhi <[email protected]> Co-authored-by: Kallol Roy <[email protected]> Co-authored-by: Arvinth C <[email protected]> Co-authored-by: Deepa Kumaraswamy <[email protected]> Co-authored-by: Durga Sarat Chandra Maddu <[email protected]> Co-authored-by: Rick Marry <[email protected]> Co-authored-by: Venkatesh-rengasamy <[email protected]> Co-authored-by: Shaik Mudassir <[email protected]> Co-authored-by: vivekshankar1 <[email protected]> Co-authored-by: Vivek Shankar <[email protected]> Co-authored-by: Vivek Yadav <[email protected]> Co-authored-by: shivangisingh <[email protected]> Co-authored-by: shaik80 <[email protected]> Co-authored-by: Chef Expeditor <[email protected]> Co-authored-by: punitmundra <[email protected]>
chef · Mar 18, 2022 · 1a98306 · 1a98306
1 parent b9f61d3
commit 1a98306
Show file tree

Hide file tree

Showing 5 changed files with 79 additions and 4 deletions.
diff --git a/components/automate-deployment/pkg/assets/binds.go b/components/automate-deployment/pkg/assets/binds.go
@@ -74,8 +74,9 @@ config-mgmt-service REQUIRED automate-es-gateway automate-pg-gateway pg-sidecar-
 config-mgmt-service BINDING_MODE strict
 data-feed-service REQUIRED automate-pg-gateway cereal-service compliance-service config-mgmt-service pg-sidecar-service secrets-service
 data-feed-service BINDING_MODE strict
-es-sidecar-service REQUIRED automate-elasticsearch automate-es-gateway
-es-sidecar-service BINDING_MODE strict
+es-sidecar-service REQUIRED automate-es-gateway
+es-sidecar-service OPTIONAL automate-elasticsearch
+es-sidecar-service BINDING_MODE relaxed
 event-feed-service REQUIRED authz-service automate-es-gateway cereal-service
 event-feed-service OPTIONAL es-sidecar-service
 event-feed-service BINDING_MODE strict

diff --git a/components/automate-deployment/pkg/constants/constants.go b/components/automate-deployment/pkg/constants/constants.go
@@ -10,3 +10,8 @@ const (
 	// RootCertPath for deployment-service
 	RootCertPath = "/hab/svc/deployment-service/data/root.crt"
 )
+
+const (
+	AutomatePGService     = "automate-postgresql"
+	AutomateSearchService = "automate-elasticsearch"
+)
diff --git a/components/automate-deployment/pkg/converge/compiler.go b/components/automate-deployment/pkg/converge/compiler.go
@@ -521,7 +521,14 @@ func (phase *RunningPhase) Run(writer *eventWriter) error {
 }
 
 func configWriteRequiresReload(pkg habpkg.Installable) bool {
-	return pkg.Name() == postgresqlServiceName || pkg.Name() == elasticsearchServiceName
+	//Adding ESGateway and PGGateway as well to the list, to resolve the issue with the service restarts
+	//while switching from internal to external and vice-versa. Althoough, the services were restarting
+	//for PG, it has been added here as a false-safe so that in the future if the services that have
+	//store the PG information in the _a2_plafrm.json file and have a direct dependency on PG will restart
+	//even if we remove the dependency in the future. Adding the GW to the list will ensure that the
+	//mitigation is executed for the GW as well, therefore restarting all the required services.
+
+	return pkg.Name() == postgresqlServiceName || pkg.Name() == elasticsearchServiceName || pkg.Name() == esGatewayServiceName || pkg.Name() == pgGatewayServiceName
 }
 
 func configOutOfDate(step runningPhaseStep) (bool, error) {

diff --git a/components/automate-deployment/pkg/server/server.go b/components/automate-deployment/pkg/server/server.go
@@ -306,6 +306,12 @@ func (s *server) buildDesiredState() (*converge.DesiredState, error) {
 		return nil, err
 	}
 
+	enableExternalPg := s.deployment.Config.GetGlobal().GetV1().GetExternal().GetPostgresql().GetEnable().GetValue()
+	logrus.Debugln("Is External PG enabled : ", enableExternalPg)
+
+	enableExternalES := s.deployment.Config.GetGlobal().GetV1().GetExternal().GetElasticsearch().GetEnable().GetValue()
+	logrus.Debugln("Is External ES enabled : ", enableExternalES)
+
 	for i, service := range expectedServices {
 		var convergeState converge.ServiceConvergeState
 		pkg := manifest.InstallableFromManifest(m, service.Name())
@@ -327,6 +333,25 @@ func (s *server) buildDesiredState() (*converge.DesiredState, error) {
 			}).Debug("Found hart override")
 		}
 
+		if service.Name() == constants.AutomatePGService {
+			if enableExternalPg {
+				service.DeploymentState = deployment.Removed
+			} else if service.DeploymentState == deployment.Removed {
+				service.DeploymentState = deployment.Running
+			}
+		}
+
+		if service.Name() == constants.AutomateSearchService {
+			if enableExternalES {
+				service.DeploymentState = deployment.Removed
+			} else if service.DeploymentState == deployment.Removed {
+				service.DeploymentState = deployment.Running
+			}
+		}
+
+		// logrus.Debugln("BUILD_DESIRED_STATE SERVICE_NAME ::: ", service.Name())
+		// logrus.Debugln("BUILD_DESIRED_STATE SERVICE_STATE ::: ", service.DeploymentState)
+
 		switch service.DeploymentState {
 		case deployment.Skip:
 			convergeState = converge.Skip()
@@ -539,8 +564,10 @@ func (s *server) doDeploySome(serviceNames []string,
 	go func(serviceNames []string) {
 		defer sender.TaskComplete()
 		eDeploy.waitForConverge(task)
+
 		// NOTE(ssd) 2018-01-25: We don't use the timeout from
 		// the request because a deploy outlives the request
+
 		eDeploy.ensureStatus(context.Background(), serviceNames, s.ensureStatusTimeout)
 		if !usedBootstrapBundle {
 			eDeploy.maybeCreateInitialUser(serviceNames)
@@ -864,13 +891,18 @@ func (s *errDeployer) ensureStatus(ctx context.Context, serviceList []string, ti
 	if s.err != nil {
 		return
 	}
+
 	e := s.sender
 
 	logctx := logrus.WithFields(logrus.Fields{
 		"mod":     "server.ensureStatus",
 		"timeout": timeout,
 	})
 
+	//To remove internal services from health check in case External ES or PG is enabled
+
+	serviceList = skipServicesForHealthCheck(serviceList, s, logctx)
+
 	var status *api.ServiceStatus
 	e.Phase(api.Running, events.CheckingServiceHealth)
 	logctx.Debug("waiting for all services to be healthy")
@@ -909,6 +941,30 @@ func (s *errDeployer) ensureStatus(ctx context.Context, serviceList []string, ti
 	logctx.Debug("all services healthy")
 }
 
+func skipServicesForHealthCheck(serviceList []string, s *errDeployer, logctx *logrus.Entry) []string {
+	servicesToSkip := make(map[string]bool)
+
+	enableExternalPg := s.deployment.Config.GetGlobal().GetV1().GetExternal().GetPostgresql().GetEnable().GetValue()
+	if enableExternalPg {
+		logctx.Debug("External PG is enabled.")
+		servicesToSkip[constants.AutomatePGService] = true
+	}
+	enableExternalEs := s.deployment.Config.GetGlobal().GetV1().GetExternal().GetElasticsearch().GetEnable().GetValue()
+
+	if enableExternalEs {
+		logctx.Debug("External ES is enabled.")
+		servicesToSkip[constants.AutomateSearchService] = true
+	}
+
+	for i, v := range serviceList {
+		if _, ok := servicesToSkip[v]; ok {
+			logctx.Debug("Removed service " + v + " from Health Check")
+			serviceList = append(serviceList[:i], serviceList[i+1:]...)
+		}
+	}
+	return serviceList
+}
+
 func (s *server) Ping(context.Context, *api.PingRequest) (*api.PingResponse, error) {
 	return &api.PingResponse{}, nil
 }

diff --git a/components/es-sidecar-service/habitat/plan.sh b/components/es-sidecar-service/habitat/plan.sh
@@ -9,10 +9,16 @@ pkg_version="1.0.0"
 pkg_maintainer="Chef Software Inc. <[email protected]>"
 pkg_license=('Chef-MLSA')
 pkg_upstream_url="http://github.com/chef/automate/components/es-sidecar-service"
+
+chef_automate_hab_binding_mode="relaxed"
+
 pkg_binds=(
-  [automate-elasticsearch]="deprecated_external_es"
   [automate-es-gateway]="http-port"
 )
+
+pkg_binds_optional=(
+  [automate-elasticsearch]="deprecated_external_es"
+)
 pkg_exports=(
   [port]=service.port
 )