From 119f2bbe4dcb6421b93645b39fc22a3db1437964 Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Sat, 2 Dec 2023 15:42:31 +1100 Subject: [PATCH] feat: initial pass at service cleanup --- cmd/identify_ingress_test.go | 4 + cmd/identify_lagoonservices.go | 70 +++---- cmd/identify_lagoonservices_test.go | 262 +++++++++----------------- legacy/build-deploy-docker-compose.sh | 116 ++++++++++++ 4 files changed, 238 insertions(+), 214 deletions(-) diff --git a/cmd/identify_ingress_test.go b/cmd/identify_ingress_test.go index 0f3ea1e2..c6dc334d 100644 --- a/cmd/identify_ingress_test.go +++ b/cmd/identify_ingress_test.go @@ -383,6 +383,10 @@ func TestIdentifyRoute(t *testing.T) { if string(retJSON) != tt.wantJSON { t.Errorf("returned autogen %v doesn't match want %v", string(retJSON), tt.wantJSON) } + t.Cleanup(func() { + helpers.UnsetEnvVars(nil) + helpers.UnsetEnvVars(tt.args.BuildPodVariables) + }) }) } } diff --git a/cmd/identify_lagoonservices.go b/cmd/identify_lagoonservices.go index d17b8ad9..64ad3420 100644 --- a/cmd/identify_lagoonservices.go +++ b/cmd/identify_lagoonservices.go @@ -1,7 +1,6 @@ package cmd import ( - "encoding/base64" "encoding/json" "fmt" @@ -11,18 +10,9 @@ import ( ) type identifyServices struct { - Name string `json:"name"` - Type string `json:"type"` - Containers []containers `json:"containers,omitempty"` -} - -type containers struct { - Name string `json:"name"` - Ports []ports `json:"ports,omitempty"` -} - -type ports struct { - Port int32 `json:"port"` + Deployments []string `json:"deployments,omitempty"` + Volumes []string `json:"volumes,omitempty"` + Services []string `json:"services,omitempty"` } var lagoonServiceIdentify = &cobra.Command{ @@ -30,7 +20,7 @@ var lagoonServiceIdentify = &cobra.Command{ Aliases: []string{"ls"}, Short: "Identify the lagoon services for a Lagoon build", RunE: func(cmd *cobra.Command, args []string) error { - gen, err := generator.GenerateInput(*rootCmd, true) + gen, err := generator.GenerateInput(*rootCmd, false) if err != nil { return err } @@ -38,22 +28,17 @@ var lagoonServiceIdentify = &cobra.Command{ if err != nil { return fmt.Errorf("error reading images flag: %v", err) } - var imageRefs struct { - Images map[string]string `json:"images"` - } - imagesStr, err := base64.StdEncoding.DecodeString(images) + imageRefs, err := loadImagesFromFile(images) if err != nil { - return fmt.Errorf("error decoding images payload: %v", err) - } - if err := json.Unmarshal(imagesStr, &imageRefs); err != nil { - return fmt.Errorf("error unmarshalling images payload: %v", err) + return err } gen.ImageReferences = imageRefs.Images out, err := LagoonServiceTemplateIdentification(gen) if err != nil { return err } - fmt.Println(out) + b, _ := json.Marshal(out) + fmt.Println(string(b)) return nil }, } @@ -62,9 +47,9 @@ var lagoonServiceIdentify = &cobra.Command{ // about the services that lagoon will be deploying (this will be kubernetes `kind: deployment`, but lagoon calls them services ¯\_(ツ)_/¯) // this command can be used to identify services that are deployed by the build, so that services that may remain in the environment can be identified // and eventually removed -func LagoonServiceTemplateIdentification(g generator.GeneratorInput) ([]identifyServices, error) { +func LagoonServiceTemplateIdentification(g generator.GeneratorInput) (*identifyServices, error) { - lServices := []identifyServices{} + servicesData := identifyServices{} lagoonBuild, err := generator.NewGenerator( g, ) @@ -74,27 +59,26 @@ func LagoonServiceTemplateIdentification(g generator.GeneratorInput) ([]identify deployments, err := servicestemplates.GenerateDeploymentTemplate(*lagoonBuild.BuildValues) if err != nil { - return nil, fmt.Errorf("couldn't generate template: %v", err) + return nil, fmt.Errorf("couldn't identify deployments: %v", err) } for _, d := range deployments { - dcs := []containers{} - for _, dc := range d.Spec.Template.Spec.Containers { - dcp := []ports{} - for _, p := range dc.Ports { - dcp = append(dcp, ports{Port: p.ContainerPort}) - } - dcs = append(dcs, containers{ - Name: dc.Name, - Ports: dcp, - }) - } - lServices = append(lServices, identifyServices{ - Name: d.Name, - Type: d.ObjectMeta.Labels["lagoon.sh/service-type"], - Containers: dcs, - }) + servicesData.Deployments = append(servicesData.Deployments, d.ObjectMeta.Name) + } + pvcs, err := servicestemplates.GeneratePVCTemplate(*lagoonBuild.BuildValues) + if err != nil { + return nil, fmt.Errorf("couldn't identify volumes: %v", err) + } + for _, pvc := range pvcs { + servicesData.Volumes = append(servicesData.Volumes, pvc.ObjectMeta.Name) + } + services, err := servicestemplates.GenerateServiceTemplate(*lagoonBuild.BuildValues) + if err != nil { + return nil, fmt.Errorf("couldn't identify services: %v", err) + } + for _, service := range services { + servicesData.Services = append(servicesData.Services, service.ObjectMeta.Name) } - return lServices, nil + return &servicesData, nil } func init() { diff --git a/cmd/identify_lagoonservices_test.go b/cmd/identify_lagoonservices_test.go index 59521a60..e2303a58 100644 --- a/cmd/identify_lagoonservices_test.go +++ b/cmd/identify_lagoonservices_test.go @@ -1,10 +1,12 @@ package cmd import ( + "encoding/json" "os" "reflect" "testing" + "github.com/andreyvit/diff" "github.com/uselagoon/build-deploy-tool/internal/dbaasclient" "github.com/uselagoon/build-deploy-tool/internal/helpers" "github.com/uselagoon/build-deploy-tool/internal/lagoon" @@ -19,7 +21,7 @@ func TestIdentifyLagoonServices(t *testing.T) { name string description string args testdata.TestData - want []identifyServices + want *identifyServices }{ { name: "test1 basic deployment", @@ -33,20 +35,12 @@ func TestIdentifyLagoonServices(t *testing.T) { "node": "harbor.example/example-project/main/node@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8", }, }, true), - want: []identifyServices{ - { - Name: "node", - Type: "basic", - Containers: []containers{ - { - Name: "basic", - Ports: []ports{ - {Port: 1234}, - {Port: 8191}, - {Port: 9001}, - }, - }, - }, + want: &identifyServices{ + Deployments: []string{ + "node", + }, + Services: []string{ + "node", }, }, }, @@ -66,59 +60,20 @@ func TestIdentifyLagoonServices(t *testing.T) { "varnish": "harbor.example/example-project/main/varnish@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8", }, }, true), - want: []identifyServices{ - { - Name: "cli", - Type: "cli-persistent", - Containers: []containers{ - { - Name: "cli", - Ports: []ports{}, - }, - }, + want: &identifyServices{ + Deployments: []string{ + "cli", + "redis", + "varnish", + "nginx-php", }, - { - Name: "redis", - Type: "redis", - Containers: []containers{ - { - Name: "redis", - Ports: []ports{ - {Port: 6379}, - }, - }, - }, + Volumes: []string{ + "nginx-php", }, - { - Name: "varnish", - Type: "varnish", - Containers: []containers{ - { - Name: "varnish", - Ports: []ports{ - {Port: 8080}, - {Port: 6082}, - }, - }, - }, - }, - { - Name: "nginx-php", - Type: "nginx-php-persistent", - Containers: []containers{ - { - Name: "nginx", - Ports: []ports{ - {Port: 8080}, - }, - }, - { - Name: "php", - Ports: []ports{ - {Port: 9000}, - }, - }, - }, + Services: []string{ + "redis", + "varnish", + "nginx-php", }, }, }, @@ -145,59 +100,20 @@ func TestIdentifyLagoonServices(t *testing.T) { }, }, }, true), - want: []identifyServices{ - { - Name: "cli", - Type: "cli-persistent", - Containers: []containers{ - { - Name: "cli", - Ports: []ports{}, - }, - }, + want: &identifyServices{ + Deployments: []string{ + "cli", + "redis", + "varnish", + "nginx-php", }, - { - Name: "redis", - Type: "redis", - Containers: []containers{ - { - Name: "redis", - Ports: []ports{ - {Port: 6379}, - }, - }, - }, + Volumes: []string{ + "nginx-php", }, - { - Name: "varnish", - Type: "varnish", - Containers: []containers{ - { - Name: "varnish", - Ports: []ports{ - {Port: 8080}, - {Port: 6082}, - }, - }, - }, - }, - { - Name: "nginx-php", - Type: "nginx-php-persistent", - Containers: []containers{ - { - Name: "nginx", - Ports: []ports{ - {Port: 8080}, - }, - }, - { - Name: "php", - Ports: []ports{ - {Port: 9000}, - }, - }, - }, + Services: []string{ + "redis", + "varnish", + "nginx-php", }, }, }, @@ -223,44 +139,19 @@ func TestIdentifyLagoonServices(t *testing.T) { }, }, }, true), - want: []identifyServices{ - { - Name: "lnd", - Type: "basic-persistent", - Containers: []containers{ - { - Name: "basic", - Ports: []ports{ - {Port: 8080}, - {Port: 10009}, - }, - }, - }, + want: &identifyServices{ + Deployments: []string{ + "lnd", + "thunderhub", + "tor", }, - { - Name: "thunderhub", - Type: "basic-persistent", - Containers: []containers{ - { - Name: "basic", - Ports: []ports{ - {Port: 3000}, - }, - }, - }, + Volumes: []string{ + "lnd", }, - { - Name: "tor", - Type: "basic", - Containers: []containers{ - { - Name: "basic", - Ports: []ports{ - {Port: 9050}, - {Port: 9051}, - }, - }, - }, + Services: []string{ + "lnd", + "thunderhub", + "tor", }, }, }, @@ -285,25 +176,52 @@ func TestIdentifyLagoonServices(t *testing.T) { }, }, }, true), - want: []identifyServices{ - { - Name: "lnd", - Type: "basic-persistent", - Containers: []containers{ - {Name: "basic", - Ports: []ports{ - {Port: 8080}, - {Port: 10009}, - }}, - }, + want: &identifyServices{ + Deployments: []string{ + "lnd", + "tor", + }, + Volumes: []string{ + "lnd", }, - { - Name: "tor", - Type: "worker-persistent", - Containers: []containers{ - {Name: "worker", - Ports: []ports{}}, + Services: []string{ + "lnd", + }, + }, + }, + + { + name: "test5-complex-custom-volumes", + args: testdata.GetSeedData( + testdata.TestData{ + ProjectName: "example-project", + EnvironmentName: "main", + Branch: "main", + BuildType: "branch", + LagoonYAML: "internal/testdata/complex/lagoon.multiple-volumes.yml", + ImageReferences: map[string]string{ + "nginx": "harbor.example/example-project/main/nginx@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8", + "php": "harbor.example/example-project/main/php@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8", + "cli": "harbor.example/example-project/main/cli@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8", + "nginx2": "harbor.example/example-project/main/nginx2@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8", + "php2": "harbor.example/example-project/main/php2@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8", + "mariadb": "harbor.example/example-project/main/mariadb@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8", }, + }, true), + want: &identifyServices{ + Deployments: []string{ + "cli", + "mariadb", + "nginx", + }, + Volumes: []string{ + "mariadb", + "nginx", + "custom-files", + }, + Services: []string{ + "mariadb", + "nginx", }, }, }, @@ -337,7 +255,9 @@ func TestIdentifyLagoonServices(t *testing.T) { t.Errorf("%v", err) } if !reflect.DeepEqual(out, tt.want) { - t.Errorf("returned output %v doesn't match want %v", out, tt.want) + r1, _ := json.MarshalIndent(out, "", " ") + s1, _ := json.MarshalIndent(tt.want, "", " ") + t.Errorf("LagoonServiceTemplateIdentification() = \n%v", diff.LineDiff(string(r1), string(s1))) } t.Cleanup(func() { helpers.UnsetEnvVars(nil) diff --git a/legacy/build-deploy-docker-compose.sh b/legacy/build-deploy-docker-compose.sh index 9e11fe6a..11b70007 100755 --- a/legacy/build-deploy-docker-compose.sh +++ b/legacy/build-deploy-docker-compose.sh @@ -1610,6 +1610,122 @@ done currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")" patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "deploymentApplyComplete" "Applying Deployments" "false" previousStepEnd=${currentStepEnd} +beginBuildStep "Service/Deployment Cleanup" "cleanupServices" + +############################################## +### CLEANUP services which have been removed from docker-compose.yaml +##############################################s + +set +x +# collect the current routes, its possible to exclude ingress by adding a label 'route.lagoon.sh/remove=false' and it won't get deleted +CURRENT_DEPLOYMENTS=$(kubectl -n ${NAMESPACE} get deployments -l "lagoon.sh/service-type" -l "lagoon.sh/service" --no-headers | cut -d " " -f 1 | xargs) +CURRENT_PVCS=$(kubectl -n ${NAMESPACE} get pvc -l "lagoon.sh/service-type" --no-headers | cut -d " " -f 1 | xargs) +CURRENT_SERVICES=$(kubectl -n ${NAMESPACE} get services -l "lagoon.sh/service-type" -l "lagoon.sh/service" --no-headers | cut -d " " -f 1 | xargs) +# collect the routes that Lagoon thinks it should have based on the .lagoon.yml and any routes that have come from the api +# using the build-deploy-tool generator +LAGOON_DEPLOYMENTS_TO_JSON=$(build-deploy-tool identify lagoon-services --images /kubectl-build-deploy/images.yaml | jq -r ) + +MATCHED_DEPLOYMENT=false +DELETE_DEPLOYMENT=() +for EXIST_DEPLOYMENT in ${CURRENT_DEPLOYMENTS}; do + for DEPLOYMENT in $(echo "$LAGOON_DEPLOYMENTS_TO_JSON" | jq -rc '.deployments[]?') + do + if [ "${EXIST_DEPLOYMENT}" == "${DEPLOYMENT}" ]; then + MATCHED_DEPLOYMENT=true + continue + fi + done + if [ "${MATCHED_DEPLOYMENT}" != "true" ]; then + DELETE_DEPLOYMENT+=($EXIST_DEPLOYMENT) + fi + MATCHED_DEPLOYMENT=false +done +MATCHED_VOLUME=false +DELETE_VOLUME=() +for EXIST_PVC in ${CURRENT_PVCS}; do + for VOLUME in $(echo "$LAGOON_DEPLOYMENTS_TO_JSON" | jq -rc '.volumes[]?') + do + if [ "${EXIST_PVC}" == "${VOLUME}" ]; then + MATCHED_VOLUME=true + continue + fi + done + if [ "${MATCHED_VOLUME}" != "true" ]; then + DELETE_VOLUME+=($EXIST_PVC) + fi + MATCHED_VOLUME=false +done +MATCHED_SERVICE=false +DELETE_SERVICE=() +for EXIST_SERVICE in ${CURRENT_SERVICES}; do + for SERVICE in $(echo "$LAGOON_DEPLOYMENTS_TO_JSON" | jq -rc '.services[]?') + do + if [ "${EXIST_SERVICE}" == "${SERVICE}" ]; then + MATCHED_SERVICE=true + continue + fi + done + if [ "${MATCHED_SERVICE}" != "true" ]; then + DELETE_SERVICE+=($EXIST_SERVICE) + fi + MATCHED_SERVICE=false +done + +SERVICE_CLEANUP_WARNINGS="false" +if [[ ${#DELETE_DEPLOYMENT[@]} -ne 0 ]] || [[ ${#DELETE_SERVICE[@]} -ne 0 ]]|| [[ ${#DELETE_VOLUME[@]} -ne 0 ]] ; then + SERVICE_CLEANUP_WARNINGS="true" + ((++BUILD_WARNING_COUNT)) + echo ">> Lagoon detected services or volumes that have been removed from the docker-compose file" + if [ "$(featureFlag CLEANUP_REMOVED_LAGOON_SERVICES)" != enabled ]; then + echo "> If you are no longer using these, then they should be removed from the environment." + echo " You can remove these in the next build by setting the flag 'LAGOON_FEATURE_FLAG_CLEANUP_REMOVED_LAGOON_SERVICES=enabled' as a GLOBAL scoped variable to this environment or project." + echo " Removing unused resources could be a dangerous operation if your application may still be referencing them." + echo " Ensure your application is no longer configured to use these resources before removing them." + echo " If you're not sure if you're using them, contact your support team." + else + echo "> The flag 'LAGOON_FEATURE_FLAG_CLEANUP_REMOVED_LAGOON_SERVICES' is enabled." + echo " Resources that were removed from the docker-compose file will now be removed from the environment." + fi + for DD in ${DELETE_DEPLOYMENT[@]} + do + if [ "$(featureFlag CLEANUP_REMOVED_LAGOON_SERVICES)" = enabled ]; then + echo ">> Removing deployment ${DD}" + if kubectl -n ${NAMESPACE} get deployments ${DD} &> /dev/null; then + kubectl -n ${NAMESPACE} delete deployment ${DD} + fi + else + echo ">> Would remove deployment ${DD}" + fi + done + for DD in ${DELETE_SERVICE[@]} + do + if [ "$(featureFlag CLEANUP_REMOVED_LAGOON_SERVICES)" = enabled ]; then + echo ">> Removing service ${DD}" + if kubectl -n ${NAMESPACE} get services ${DD} &> /dev/null; then + kubectl -n ${NAMESPACE} delete service ${DD} + fi + else + echo ">> Would remove service ${DD}" + fi + done + for DD in ${DELETE_VOLUME[@]} + do + if [ "$(featureFlag CLEANUP_REMOVED_LAGOON_SERVICES)" = enabled ]; then + echo ">> Removing volume ${DD}" + if kubectl -n ${NAMESPACE} get pvc ${DD} &> /dev/null; then + kubectl -n ${NAMESPACE} delete pvc ${DD} + fi + else + echo ">> Would remove pvc ${DD}" + fi + done +else + echo "No service cleanup required" +fi + +currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")" +patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "serviceCleanupComplete" "Service/Deployment Cleanup" "${SERVICE_CLEANUP_WARNINGS}" +previousStepEnd=${currentStepEnd} beginBuildStep "Cronjob Cleanup" "cleaningUpCronjobs" ##############################################