diff --git a/cmd/cluster.go b/cmd/cluster.go index dc2f490..dbf8179 100644 --- a/cmd/cluster.go +++ b/cmd/cluster.go @@ -374,10 +374,10 @@ func installGitServer() { "--name=git-server", "-v"+clusterRootPath+":/git-server/repos", "-v"+clusterRootPath+"/.ssh:/git-server/keys", - "jkarlos/git-server-docker", - ) + "jkarlos/git-server-docker") err := cmd.Run() + if err != nil { log.Fatalf("failed to start git server: %v", err) } @@ -464,7 +464,8 @@ func installSealedSecrets() { } // reload secret if exists - secretsPath := filepath.Join(clusterRootPath, ".secrets", "sealed-secrets.yaml") + secretsDir := filepath.Join(clusterRootPath, ".secrets") + secretsPath := filepath.Join(secretsDir, "sealed-secrets.yaml") if _, err = os.Stat(secretsPath); os.IsExist(err) { secret, err := ioutil.ReadFile(secretsPath) check(err) @@ -495,6 +496,8 @@ func installSealedSecrets() { secret, err := cmd.Output() check(err) + os.MkdirAll(secretsDir, 0755) //if .secrets folder is missing, create + log.Info("Trying to write new file") err = ioutil.WriteFile(secretsPath, secret, 0644) check(err) } @@ -561,7 +564,7 @@ func installFluxHelmOperator() { err := cmd.Run() if err != nil { - log.Fatal("failed to start flux helm operator: %v", err) + log.Fatalf("failed to start flux helm operator: %v", err) } } diff --git a/docs/cluster.md b/docs/cluster.md index bcd4d41..20d3f2e 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -94,7 +94,7 @@ git commit -m "modified podinfo deployment" 3. Within the context of the deploy folder, preform a git add and commit, eg. ```sh git add -A -git commit -m "modified podinfo deployment" +git commit -m "removed podinfo deployment" ``` 4. Your changes will take effect in the cluster within a few minutes diff --git a/docs/cluster_troubleshoot.md b/docs/cluster_troubleshoot.md new file mode 100644 index 0000000..b998da1 --- /dev/null +++ b/docs/cluster_troubleshoot.md @@ -0,0 +1,62 @@ +# Condo Cluster Deployment Troubleshooting +While Kubernetes combined with the [Git-ops](https://www.weave.works/technologies/gitops/) approach provides a multitude of advantages in terms of scalability and maintainability, can be complicated to set up and diagnose. Below are common commands used to gather more information about your deployments. + + +### [Command Set 1] Pods within your cluster +A [pod](https://kubernetes.io/docs/concepts/workloads/pods/) is an instance of your running application that has been deployed (although it may still be experiencing issues) +```sh +kubectl get pods {pod-name} -n {namespace} #Get pod within a particular namespace with a particular name +kubectl get pods -n {namespace} #Get pods within a particular namespace +kubectl get pods --all-namespaces #Get all pods within the entire cluster + +kubectl describe {pod-name} -n {namespace} #Get more details of a particular pod + +kubectl logs {pod-name} #Get log dump of a particular pod +``` + +### [Command Set 2] Deployments within your cluster +A [deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) defines your desired state (configuration) of your application in the cluster. +```sh +kubectl get deployments {deployment-name} -n {namespace} #Get pod within a particular namespace with a particular name +kubectl get deployments -n {namespace} #Get deployments within a particular namespace +kubectl get deployments --all-namespaces #Get all deployments within the entire cluster + +kubectl describe {deployment-name} -n {namespace} #Get more details of a particular deployment +``` + +## Common Issues +### Where is my pod? +After performing a git add and commit to your deployment yaml file within the deploy folder (`UserRoot/.am/clusters/{your-cluster-name}/deploy/`), it usually takes a few minutes for flux to detect and role out the change. The easiest way to check the status of your pod(s) is by finding the pod by the namespace it was supposed to be in (see command 1 above). + +--- + +### I see my pod(s) but they are not ready/show an error +Pods do take time to build, however, they can fail to start or crash during runtime due to a number of reasons. To get more details as to the state of the pod(s) use the `describe` command (see command 1 above). For more in-depth information, consider getting the logs for that pod (see command 1 above). + +--- + +### I have waited 10 minutes, I still do not see my pod +Check to see if flux created a deployment for your application (see command 1 above). + +If the deployment exists but pods are not being created, try scaling the pods manually with the following command: +```sh +kubectl scale deployment {deployment-name} -n {namespace} --replicas=2 +``` +A deployment may have been created but pods were not automatically scaled due to default helm configurations. + +--- + +### I do not see the deployment either +If the deployment does not exist, investigate the flux logs to determine why flux did not create the deployment by using the following commands: + +```sh +kubectl get pods -n weave #List all of the pods in the weave namespace + +#Identify the flux pod, usually a pod like: flux-7785dfc54d-cx9w2 + +kubectl logs {flux pod} -n weave #Get logs from flux + +#(OPTIONAL) +kubectl logs flux-7785dfc54d-cx9w2 -n weave > flux-logs.txt #pipe flux logs to a text file +``` + diff --git a/internal/git/gitService.go b/internal/git/gitService.go index 9013078..a4b010e 100644 --- a/internal/git/gitService.go +++ b/internal/git/gitService.go @@ -91,12 +91,17 @@ func setUpLocalGitFolder(folderName string, clusterRootPath string, clusterName func moveCloneIntoLocalRepo(folderName string, clusterRootPath string) { - commandRmGit := exec.Command("rm", "-rf", ".git") - commandRmGit.Dir = clusterRootPath + FPS + "tmp" + FPS + folderName - errRmGit := commandRmGit.Run() - if errRmGit != nil { - log.Fatalf("Error removing git folder at /tmp/"+folderName+". %s", errRmGit) - } + // commandRmGit := exec.Command("rm", "-rf", ".git") + // commandRmGit.Dir = clusterRootPath + FPS + "tmp" + FPS + folderName + // errRmGit := commandRmGit.Run() + // if errRmGit != nil { + // log.Fatalf("Error removing git folder at /tmp/"+folderName+". %s", errRmGit) + // } + + removeGitError := os.RemoveAll(clusterRootPath + FPS + "tmp" + FPS + folderName + FPS + ".git") + if removeGitError != nil { + log.Fatal(removeGitError) + } errMove := os.Rename(clusterRootPath+FPS+"tmp"+FPS+folderName, clusterRootPath+FPS+folderName)