From 5bef872aba87f9050bc240d1a7526331e1674f12 Mon Sep 17 00:00:00 2001 From: Clifford Cheefoon Date: Fri, 3 Sep 2021 15:11:35 -0400 Subject: [PATCH 1/3] CHECKPOINT --- docs/cluster.md | 2 +- docs/cluster_troubleshoot.md | 62 ++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 docs/cluster_troubleshoot.md diff --git a/docs/cluster.md b/docs/cluster.md index bcd4d41..20d3f2e 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -94,7 +94,7 @@ git commit -m "modified podinfo deployment" 3. Within the context of the deploy folder, preform a git add and commit, eg. ```sh git add -A -git commit -m "modified podinfo deployment" +git commit -m "removed podinfo deployment" ``` 4. Your changes will take effect in the cluster within a few minutes diff --git a/docs/cluster_troubleshoot.md b/docs/cluster_troubleshoot.md new file mode 100644 index 0000000..b998da1 --- /dev/null +++ b/docs/cluster_troubleshoot.md @@ -0,0 +1,62 @@ +# Condo Cluster Deployment Troubleshooting +While Kubernetes combined with the [Git-ops](https://www.weave.works/technologies/gitops/) approach provides a multitude of advantages in terms of scalability and maintainability, can be complicated to set up and diagnose. Below are common commands used to gather more information about your deployments. + + +### [Command Set 1] Pods within your cluster +A [pod](https://kubernetes.io/docs/concepts/workloads/pods/) is an instance of your running application that has been deployed (although it may still be experiencing issues) +```sh +kubectl get pods {pod-name} -n {namespace} #Get pod within a particular namespace with a particular name +kubectl get pods -n {namespace} #Get pods within a particular namespace +kubectl get pods --all-namespaces #Get all pods within the entire cluster + +kubectl describe {pod-name} -n {namespace} #Get more details of a particular pod + +kubectl logs {pod-name} #Get log dump of a particular pod +``` + +### [Command Set 2] Deployments within your cluster +A [deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) defines your desired state (configuration) of your application in the cluster. +```sh +kubectl get deployments {deployment-name} -n {namespace} #Get pod within a particular namespace with a particular name +kubectl get deployments -n {namespace} #Get deployments within a particular namespace +kubectl get deployments --all-namespaces #Get all deployments within the entire cluster + +kubectl describe {deployment-name} -n {namespace} #Get more details of a particular deployment +``` + +## Common Issues +### Where is my pod? +After performing a git add and commit to your deployment yaml file within the deploy folder (`UserRoot/.am/clusters/{your-cluster-name}/deploy/`), it usually takes a few minutes for flux to detect and role out the change. The easiest way to check the status of your pod(s) is by finding the pod by the namespace it was supposed to be in (see command 1 above). + +--- + +### I see my pod(s) but they are not ready/show an error +Pods do take time to build, however, they can fail to start or crash during runtime due to a number of reasons. To get more details as to the state of the pod(s) use the `describe` command (see command 1 above). For more in-depth information, consider getting the logs for that pod (see command 1 above). + +--- + +### I have waited 10 minutes, I still do not see my pod +Check to see if flux created a deployment for your application (see command 1 above). + +If the deployment exists but pods are not being created, try scaling the pods manually with the following command: +```sh +kubectl scale deployment {deployment-name} -n {namespace} --replicas=2 +``` +A deployment may have been created but pods were not automatically scaled due to default helm configurations. + +--- + +### I do not see the deployment either +If the deployment does not exist, investigate the flux logs to determine why flux did not create the deployment by using the following commands: + +```sh +kubectl get pods -n weave #List all of the pods in the weave namespace + +#Identify the flux pod, usually a pod like: flux-7785dfc54d-cx9w2 + +kubectl logs {flux pod} -n weave #Get logs from flux + +#(OPTIONAL) +kubectl logs flux-7785dfc54d-cx9w2 -n weave > flux-logs.txt #pipe flux logs to a text file +``` + From a03cb7a87daa185f6f803c5516f3f8d2952a7c4e Mon Sep 17 00:00:00 2001 From: Clifford Cheefoon Date: Tue, 14 Sep 2021 13:21:41 -0400 Subject: [PATCH 2/3] changed to use OS api --- internal/git/gitService.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/internal/git/gitService.go b/internal/git/gitService.go index 9013078..a4b010e 100644 --- a/internal/git/gitService.go +++ b/internal/git/gitService.go @@ -91,12 +91,17 @@ func setUpLocalGitFolder(folderName string, clusterRootPath string, clusterName func moveCloneIntoLocalRepo(folderName string, clusterRootPath string) { - commandRmGit := exec.Command("rm", "-rf", ".git") - commandRmGit.Dir = clusterRootPath + FPS + "tmp" + FPS + folderName - errRmGit := commandRmGit.Run() - if errRmGit != nil { - log.Fatalf("Error removing git folder at /tmp/"+folderName+". %s", errRmGit) - } + // commandRmGit := exec.Command("rm", "-rf", ".git") + // commandRmGit.Dir = clusterRootPath + FPS + "tmp" + FPS + folderName + // errRmGit := commandRmGit.Run() + // if errRmGit != nil { + // log.Fatalf("Error removing git folder at /tmp/"+folderName+". %s", errRmGit) + // } + + removeGitError := os.RemoveAll(clusterRootPath + FPS + "tmp" + FPS + folderName + FPS + ".git") + if removeGitError != nil { + log.Fatal(removeGitError) + } errMove := os.Rename(clusterRootPath+FPS+"tmp"+FPS+folderName, clusterRootPath+FPS+folderName) From 5afbe469d6ae8b7364aba9801086538f12933b64 Mon Sep 17 00:00:00 2001 From: Clifford Cheefoon Date: Wed, 20 Oct 2021 17:00:45 -0400 Subject: [PATCH 3/3] .secret folder creation before file write --- cmd/cluster.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/cluster.go b/cmd/cluster.go index dc2f490..dbf8179 100644 --- a/cmd/cluster.go +++ b/cmd/cluster.go @@ -374,10 +374,10 @@ func installGitServer() { "--name=git-server", "-v"+clusterRootPath+":/git-server/repos", "-v"+clusterRootPath+"/.ssh:/git-server/keys", - "jkarlos/git-server-docker", - ) + "jkarlos/git-server-docker") err := cmd.Run() + if err != nil { log.Fatalf("failed to start git server: %v", err) } @@ -464,7 +464,8 @@ func installSealedSecrets() { } // reload secret if exists - secretsPath := filepath.Join(clusterRootPath, ".secrets", "sealed-secrets.yaml") + secretsDir := filepath.Join(clusterRootPath, ".secrets") + secretsPath := filepath.Join(secretsDir, "sealed-secrets.yaml") if _, err = os.Stat(secretsPath); os.IsExist(err) { secret, err := ioutil.ReadFile(secretsPath) check(err) @@ -495,6 +496,8 @@ func installSealedSecrets() { secret, err := cmd.Output() check(err) + os.MkdirAll(secretsDir, 0755) //if .secrets folder is missing, create + log.Info("Trying to write new file") err = ioutil.WriteFile(secretsPath, secret, 0644) check(err) } @@ -561,7 +564,7 @@ func installFluxHelmOperator() { err := cmd.Run() if err != nil { - log.Fatal("failed to start flux helm operator: %v", err) + log.Fatalf("failed to start flux helm operator: %v", err) } }