Skip to content

Commit

Permalink
feat: initial pass at service cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
shreddedbacon committed Dec 3, 2023
1 parent a429aff commit de56df8
Show file tree
Hide file tree
Showing 5 changed files with 347 additions and 0 deletions.
4 changes: 4 additions & 0 deletions cmd/identify_ingress_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"time"

"github.com/uselagoon/build-deploy-tool/internal/dbaasclient"
"github.com/uselagoon/build-deploy-tool/internal/helpers"
)

func TestIdentifyRoute(t *testing.T) {
Expand Down Expand Up @@ -530,6 +531,9 @@ func TestIdentifyRoute(t *testing.T) {
if string(retJSON) != tt.wantJSON {
t.Errorf("returned autogen %v doesn't match want %v", string(retJSON), tt.wantJSON)
}
t.Cleanup(func() {
helpers.UnsetEnvVars(nil)
})
})
}
}
Expand Down
70 changes: 70 additions & 0 deletions cmd/identify_services.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package cmd

import (
"encoding/json"
"fmt"

"github.com/spf13/cobra"
generator "github.com/uselagoon/build-deploy-tool/internal/generator"
"github.com/uselagoon/build-deploy-tool/internal/helpers"
)

type identifyServices struct {
Name string `json:"name"`
Type string `json:"type"`
}

var servicesIdentify = &cobra.Command{
Use: "services",
Aliases: []string{"s"},
Short: "Identify services that this build would create",
RunE: func(cmd *cobra.Command, args []string) error {
generator, err := generatorInput(false)
if err != nil {
return err
}
ret, _, err := IdentifyServices(generator)
if err != nil {
return err
}
retJSON, _ := json.Marshal(ret)
fmt.Println(string(retJSON))
return nil
},
}

// IdentifyServices identifies services that this build would create
func IdentifyServices(g generator.GeneratorInput) ([]string, []identifyServices, error) {
lagoonBuild, err := generator.NewGenerator(
g,
)
if err != nil {
return nil, nil, err
}

services := []string{}
serviceTypes := []identifyServices{}
for _, service := range lagoonBuild.BuildValues.Services {
if service.Type != "" {
services = helpers.AppendIfMissing(services, service.OverrideName)
serviceTypes = AppendIfMissing(serviceTypes, identifyServices{
Name: service.OverrideName,
Type: service.Type,
})
}
}
return services, serviceTypes, nil
}

func init() {
identifyCmd.AddCommand(servicesIdentify)
}

func AppendIfMissing(slice []identifyServices, i identifyServices) []identifyServices {
for _, ele := range slice {
if ele.Name == i.Name {
return slice
}
}
return append(slice, i)
}
186 changes: 186 additions & 0 deletions cmd/identify_services_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,186 @@
package cmd

import (
"os"
"reflect"
"testing"
"time"

"github.com/uselagoon/build-deploy-tool/internal/dbaasclient"
"github.com/uselagoon/build-deploy-tool/internal/helpers"
)

func TestIdentifyServices(t *testing.T) {
type args struct {
alertContact string
statusPageID string
projectName string
environmentName string
branch string
prNumber string
prHeadBranch string
prBaseBranch string
environmentType string
buildType string
activeEnvironment string
standbyEnvironment string
cacheNoCache string
serviceID string
secretPrefix string
projectVars string
envVars string
lagoonVersion string
lagoonYAML string
valuesFilePath string
templatePath string
}
tests := []struct {
name string
args args
want []string
wantServices []identifyServices
wantErr bool
}{
{
name: "test1 check LAGOON_FASTLY_SERVICE_IDS with secret no values",
args: args{
alertContact: "alertcontact",
statusPageID: "statuspageid",
projectName: "example-project",
environmentName: "main",
environmentType: "production",
buildType: "branch",
lagoonVersion: "v2.7.x",
branch: "main",
projectVars: `[{"name":"LAGOON_SYSTEM_ROUTER_PATTERN","value":"${service}-${project}-${environment}.example.com","scope":"internal_system"},{"name":"LAGOON_FASTLY_SERVICE_IDS","value":"example.com:service-id:true:annotationscom","scope":"build"}]`,
envVars: `[]`,
lagoonYAML: "../test-resources/identify-ingress/test1/lagoon.yml",
templatePath: "../test-resources/output",
},
want: []string{"node"},
wantServices: []identifyServices{{Name: "node", Type: "node"}},
},
{
name: "test16 autogenerated routes where lagoon.name of service does not match service names",
args: args{
alertContact: "alertcontact",
statusPageID: "statuspageid",
projectName: "content-example-com",
environmentName: "feature-migration",
environmentType: "development",
buildType: "branch",
lagoonVersion: "v2.7.x",
branch: "feature/migration",
projectVars: `[{"name":"LAGOON_SYSTEM_ROUTER_PATTERN","value":"${environment}.${project}.example.com","scope":"internal_system"}]`,
envVars: `[]`,
lagoonYAML: "../test-resources/identify-ingress/test16/lagoon.yml",
templatePath: "../test-resources/output",
},
want: []string{"cli", "nginx-php", "mariadb", "redis"},
wantServices: []identifyServices{
{Name: "cli", Type: "cli-persistent"},
{Name: "nginx-php", Type: "nginx-php-persistent"},
{Name: "mariadb", Type: "mariadb-dbaas"},
{Name: "redis", Type: "redis"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// set the environment variables from args
err := os.Setenv("MONITORING_ALERTCONTACT", tt.args.alertContact)
if err != nil {
t.Errorf("%v", err)
}
err = os.Setenv("MONITORING_STATUSPAGEID", tt.args.statusPageID)
if err != nil {
t.Errorf("%v", err)
}
err = os.Setenv("PROJECT", tt.args.projectName)
if err != nil {
t.Errorf("%v", err)
}
err = os.Setenv("ENVIRONMENT", tt.args.environmentName)
if err != nil {
t.Errorf("%v", err)
}
err = os.Setenv("BRANCH", tt.args.branch)
if err != nil {
t.Errorf("%v", err)
}
err = os.Setenv("LAGOON_GIT_BRANCH", tt.args.branch)
if err != nil {
t.Errorf("%v", err)
}
err = os.Setenv("PR_NUMBER", tt.args.prNumber)
if err != nil {
t.Errorf("%v", err)
}
err = os.Setenv("PR_HEAD_BRANCH", tt.args.prHeadBranch)
if err != nil {
t.Errorf("%v", err)
}
err = os.Setenv("PR_BASE_BRANCH", tt.args.prBaseBranch)
if err != nil {
t.Errorf("%v", err)
}
err = os.Setenv("ENVIRONMENT_TYPE", tt.args.environmentType)
if err != nil {
t.Errorf("%v", err)
}
err = os.Setenv("BUILD_TYPE", tt.args.buildType)
if err != nil {
t.Errorf("%v", err)
}
err = os.Setenv("ACTIVE_ENVIRONMENT", tt.args.activeEnvironment)
if err != nil {
t.Errorf("%v", err)
}
err = os.Setenv("STANDBY_ENVIRONMENT", tt.args.standbyEnvironment)
if err != nil {
t.Errorf("%v", err)
}
err = os.Setenv("LAGOON_FASTLY_NOCACHE_SERVICE_ID", tt.args.cacheNoCache)
if err != nil {
t.Errorf("%v", err)
}
err = os.Setenv("LAGOON_PROJECT_VARIABLES", tt.args.projectVars)
if err != nil {
t.Errorf("%v", err)
}
err = os.Setenv("LAGOON_ENVIRONMENT_VARIABLES", tt.args.envVars)
if err != nil {
t.Errorf("%v", err)
}
err = os.Setenv("LAGOON_VERSION", tt.args.lagoonVersion)
if err != nil {
t.Errorf("%v", err)
}
generator, err := generatorInput(false)
if err != nil {
t.Errorf("%v", err)
}
generator.LagoonYAML = tt.args.lagoonYAML
// add dbaasclient overrides for tests
generator.DBaaSClient = dbaasclient.NewClient(dbaasclient.Client{
RetryMax: 5,
RetryWaitMin: time.Duration(10) * time.Millisecond,
RetryWaitMax: time.Duration(50) * time.Millisecond,
})
got, got2, err := IdentifyServices(generator)
if (err != nil) != tt.wantErr {
t.Errorf("IdentifyServices() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("IdentifyServices() = %v, want %v", got, tt.want)
}
if !reflect.DeepEqual(got2, tt.wantServices) {
t.Errorf("IdentifyServices() = %v, want %v", got2, tt.wantServices)
}
t.Cleanup(func() {
helpers.UnsetEnvVars(nil)
})
})
}
}
9 changes: 9 additions & 0 deletions internal/helpers/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -219,3 +219,12 @@ func DeepCopy(src, dist interface{}) (err error) {
}
return gob.NewDecoder(&buf).Decode(dist)
}

func AppendIfMissing(slice []string, i string) []string {
for _, ele := range slice {
if ele == i {
return slice
}
}
return append(slice, i)
}
78 changes: 78 additions & 0 deletions legacy/build-deploy-docker-compose.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1757,6 +1757,84 @@ set +x
currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "deploymentApplyComplete" "Applying Deployments" "false"
previousStepEnd=${currentStepEnd}
beginBuildStep "Service/Deployment Cleanup" "cleanupServices"

##############################################
### CLEANUP services which have been removed from docker-compose.yaml
##############################################s

set +x
# collect the current routes, its possible to exclude ingress by adding a label 'route.lagoon.sh/remove=false' and it won't get deleted
CURRENT_SERVICES=$(kubectl -n ${NAMESPACE} get deployments -l "lagoon.sh/service-type" -l "lagoon.sh/service" --no-headers | cut -d " " -f 1 | xargs)
# collect the routes that Lagoon thinks it should have based on the .lagoon.yml and any routes that have come from the api
# using the build-deploy-tool generator
SERVICES_TO_JSON=$(build-deploy-tool identify services | jq -r '.[]')

MATCHED_SERVICE=false
DELETE_SERVICE=()
# loop over the routes from kubernetes
for EXIST_SERVICE in ${CURRENT_SERVICES}; do
# loop over the routes that Lagoon thinks it should have
for SERVICE in ${SERVICES_TO_JSON}; do
if [ "${EXIST_SERVICE}" == "${SERVICE}" ]; then
MATCHED_SERVICE=true
continue
fi
done
if [ "${MATCHED_SERVICE}" != "true" ]; then
DELETE_SERVICE+=($EXIST_SERVICE)
fi
MATCHED_SERVICE=false
done

SERVICE_CLEANUP_WARNINGS="false"
if [ ${#DELETE_SERVICE[@]} -ne 0 ]; then
SERVICE_CLEANUP_WARNINGS="true"
((++BUILD_WARNING_COUNT))
echo ">> Lagoon detected services that have been removed from the docker-compose file"
if [ "$(featureFlag CLEANUP_REMOVED_LAGOON_SERVICES)" != enabled ]; then
echo "> You can remove these in the next build by setting the flag 'LAGOON_FEATURE_FLAG_CLEANUP_REMOVED_LAGOON_SERVICES=enabled' as a GLOBAL scoped variable to this environment or project"
fi
for DS in ${DELETE_SERVICE[@]}
do
if [ "$(featureFlag CLEANUP_REMOVED_LAGOON_SERVICES)" = enabled ]; then
echo ">> Removing deployment ${DS}"
if kubectl -n ${NAMESPACE} get deployments ${DS} &> /dev/null; then
kubectl -n ${NAMESPACE} delete deployment ${DS}
fi
if kubectl -n ${NAMESPACE} get service ${DS} &> /dev/null; then
echo ">>> Removing associated service ${DS}"
kubectl -n ${NAMESPACE} delete service ${DS}
fi
if kubectl -n ${NAMESPACE} get ingress ${DS} &> /dev/null; then
echo ">>> Removing associated ingress ${DS}"
kubectl -n ${NAMESPACE} delete ingress ${DS}
fi
if kubectl -n ${NAMESPACE} get pvc ${DS} &> /dev/null; then
echo ">>> Removing associated persistent volume ${DS}"
kubectl -n ${NAMESPACE} delete pvc ${DS}
fi
#delete anything else?
else
echo ">> The deployment '${DS}' would be removed"
if kubectl -n ${NAMESPACE} get service ${DS} &> /dev/null; then
echo ">>> The associated service '${DS}' would be removed"
fi
if kubectl -n ${NAMESPACE} get ingress ${DS} &> /dev/null; then
echo ">>> The associated ingress '${DS}' would be removed"
fi
if kubectl -n ${NAMESPACE} get pvc ${DS} &> /dev/null; then
echo ">>> The associated persistent volume '${DS}' would be removed"
fi
fi
done
else
echo "No service cleanup required"
fi

currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "serviceCleanupComplete" "Service/Deployment Cleanup" "${SERVICE_CLEANUP_WARNINGS}"
previousStepEnd=${currentStepEnd}
beginBuildStep "Cronjob Cleanup" "cleaningUpCronjobs"
set -x

Expand Down

0 comments on commit de56df8

Please sign in to comment.