diff --git a/Gopkg.lock b/Gopkg.lock index 22f88ed..36bd88d 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -41,6 +41,22 @@ revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" version = "v1.4.7" +[[projects]] + digest = "1:05334858a0cfb538622a066e065287f63f42bee26a7fda93a789674225057201" + name = "github.com/hashicorp/go-cleanhttp" + packages = ["."] + pruneopts = "" + revision = "e8ab9daed8d1ddd2d3c4efba338fe2eeae2e4f18" + version = "v0.5.0" + +[[projects]] + digest = "1:776139dc18d63ef223ffaca5d8e9a3057174890f84393d3c881e934100b66dbc" + name = "github.com/hashicorp/go-retryablehttp" + packages = ["."] + pruneopts = "" + revision = "73489d0a1476f0c9e6fb03f9c39241523a496dfd" + version = "v0.5.2" + [[projects]] digest = "1:d14365c51dd1d34d5c79833ec91413bfbb166be978724f15701e17080dc06dec" name = "github.com/hashicorp/hcl" @@ -235,6 +251,7 @@ input-imports = [ "github.com/MakeNowJust/heredoc", "github.com/Masterminds/semver", + "github.com/hashicorp/go-retryablehttp", "github.com/mitchellh/go-homedir", "github.com/pkg/errors", "github.com/spf13/cobra", diff --git a/Gopkg.toml b/Gopkg.toml index 8ed8578..ffdcd39 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -30,3 +30,7 @@ [[constraint]] name = "gopkg.in/yaml.v2" version = "v2.2.1" + +[[constraint]] + name = "github.com/hashicorp/go-retryablehttp" + version = "v0.5.2" diff --git a/pkg/exec/exec.go b/pkg/exec/exec.go index 10829cd..9e1a0f4 100644 --- a/pkg/exec/exec.go +++ b/pkg/exec/exec.go @@ -17,12 +17,12 @@ package exec import ( "bufio" "fmt" - "io" - "os/exec" - "strings" - "github.com/helm/chart-testing/pkg/util" "github.com/pkg/errors" + "io" + "os" + "os/exec" + "strings" ) type ProcessExecutor struct { @@ -40,14 +40,11 @@ func (p ProcessExecutor) RunProcessAndCaptureOutput(executable string, execArgs } func (p ProcessExecutor) RunProcessInDirAndCaptureOutput(workingDirectory string, executable string, execArgs ...interface{}) (string, error) { - args, err := util.Flatten(execArgs) - if p.debug { - fmt.Println(">>>", executable, strings.Join(args, " ")) - } + cmd, err := p.CreateProcess(executable, execArgs...) if err != nil { - return "", errors.Wrap(err, "Invalid arguments supplied") + return "", err } - cmd := exec.Command(executable, args...) + cmd.Dir = workingDirectory bytes, err := cmd.CombinedOutput() @@ -58,14 +55,10 @@ func (p ProcessExecutor) RunProcessInDirAndCaptureOutput(workingDirectory string } func (p ProcessExecutor) RunProcess(executable string, execArgs ...interface{}) error { - args, err := util.Flatten(execArgs) - if p.debug { - fmt.Println(">>>", executable, strings.Join(args, " ")) - } + cmd, err := p.CreateProcess(executable, execArgs...) if err != nil { - return errors.Wrap(err, "Invalid arguments supplied") + return err } - cmd := exec.Command(executable, args...) outReader, err := cmd.StdoutPipe() if err != nil { @@ -96,3 +89,45 @@ func (p ProcessExecutor) RunProcess(executable string, execArgs ...interface{}) return nil } + +func (p ProcessExecutor) CreateProcess(executable string, execArgs ...interface{}) (*exec.Cmd, error) { + args, err := util.Flatten(execArgs) + if p.debug { + fmt.Println(">>>", executable, strings.Join(args, " ")) + } + if err != nil { + return nil, errors.Wrap(err, "Invalid arguments supplied") + } + cmd := exec.Command(executable, args...) + + return cmd, nil +} + +type fn func(port int) error + +func (p ProcessExecutor) RunWithProxy(withProxy fn) error { + randomPort, err := util.GetRandomPort() + if err != nil { + return errors.Wrap(err, "Could not find a free port for running 'kubectl proxy'") + } + + fmt.Printf("Running 'kubectl proxy' on port %d\n", randomPort) + cmdProxy, err := p.CreateProcess("kubectl", "proxy", fmt.Sprintf("--port=%d", randomPort)) + if err != nil { + return errors.Wrap(err, "Error creating the 'kubectl proxy' process") + } + err = cmdProxy.Start() + if err != nil { + return errors.Wrap(err, "Error starting the 'kubectl proxy' process") + } + + err = withProxy(randomPort) + + cmdProxy.Process.Signal(os.Kill) + + if err != nil { + return errors.Wrap(err, "Error running command with proxy") + } + + return nil +} diff --git a/pkg/tool/kubectl.go b/pkg/tool/kubectl.go index 8c72765..97c287b 100644 --- a/pkg/tool/kubectl.go +++ b/pkg/tool/kubectl.go @@ -1,11 +1,14 @@ package tool import ( + "bytes" "encoding/json" "fmt" + "net/http" "strings" "time" + "github.com/hashicorp/go-retryablehttp" "github.com/helm/chart-testing/pkg/exec" "github.com/pkg/errors" ) @@ -20,24 +23,13 @@ func NewKubectl(exec exec.ProcessExecutor) Kubectl { } } -// DeleteNamespace deletes the specified namespace. If the namespace does not terminate within 90s, pods running in the +// DeleteNamespace deletes the specified namespace. If the namespace does not terminate within 120s, pods running in the // namespace and, eventually, the namespace itself are force-deleted. func (k Kubectl) DeleteNamespace(namespace string) { - - fmt.Println("Deleting pvcs...") - if err := k.exec.RunProcess("kubectl", "delete", "pvc", "--namespace", namespace, "--all"); err != nil { - fmt.Println("Error deleting pvc(s):", err) - } - - fmt.Println("Deleting pvs...") - if err := k.exec.RunProcess("kubectl", "delete", "pv", "--namespace", namespace, "--all"); err != nil { - fmt.Println("Error deleting pv(s):", err) - } - fmt.Printf("Deleting namespace '%s'...\n", namespace) timeoutSec := "120s" if err := k.exec.RunProcess("kubectl", "delete", "namespace", namespace, "--timeout", timeoutSec); err != nil { - fmt.Printf("Namespace '%s' did not terminate after %s.", namespace, timeoutSec) + fmt.Printf("Namespace '%s' did not terminate after %s.\n", namespace, timeoutSec) } if _, err := k.exec.RunProcessAndCaptureOutput("kubectl", "get", "namespace", namespace); err != nil { @@ -45,23 +37,103 @@ func (k Kubectl) DeleteNamespace(namespace string) { return } - fmt.Printf("Namespace '%s' did not terminate after %s.", namespace, timeoutSec) + fmt.Printf("Namespace '%s' did not terminate after %s.\n", namespace, timeoutSec) fmt.Println("Force-deleting pods...") if err := k.exec.RunProcess("kubectl", "delete", "pods", "--namespace", namespace, "--all", "--force", "--grace-period=0"); err != nil { fmt.Println("Error deleting pods:", err) } - time.Sleep(3 * time.Second) + fmt.Println("Force-deleting pvcs...") + if err := k.exec.RunProcess("kubectl", "delete", "pvc", "--namespace", namespace, "--all", "--force", "--grace-period=0"); err != nil { + fmt.Println("Error deleting pvc(s):", err) + } - if err := k.exec.RunProcess("kubectl", "get", "namespace", namespace); err != nil { - fmt.Printf("Force-deleting namespace '%s'...\n", namespace) - if err := k.exec.RunProcess("kubectl", "delete", "namespace", namespace, "--force", "--grace-period=0"); err != nil { - fmt.Println("Error deleting namespace:", err) + fmt.Println("Force-deleting pvs...") + if err := k.exec.RunProcess("kubectl", "delete", "pv", "--namespace", namespace, "--all", "--force", "--grace-period=0"); err != nil { + fmt.Println("Error deleting pv(s):", err) + } + + // Give it some more time to be deleted by K8s + time.Sleep(5 * time.Second) + + if _, err := k.exec.RunProcessAndCaptureOutput("kubectl", "get", "namespace", namespace); err != nil { + fmt.Printf("Namespace '%s' terminated.\n", namespace) + } else { + if err := k.forceNamespaceDeletion(namespace); err != nil { + fmt.Println("Error force deleting namespace:", err) } } } +func (k Kubectl) forceNamespaceDeletion(namespace string) error { + // Getting the namespace json to remove the finalizer + cmdOutput, err := k.exec.RunProcessAndCaptureOutput("kubectl", "get", "namespace", namespace, "--output=json") + if err != nil { + fmt.Println("Error getting namespace json:", err) + return err + } + + namespaceUpdate := map[string]interface{}{} + err = json.Unmarshal([]byte(cmdOutput), &namespaceUpdate) + if err != nil { + fmt.Println("Error in unmarshalling the payload:", err) + return err + } + namespaceUpdate["spec"] = nil + namespaceUpdateBytes, err := json.Marshal(&namespaceUpdate) + if err != nil { + fmt.Println("Error in marshalling the payload:", err) + return err + } + + // Remove finalizer from the namespace + fun := func(port int) error { + fmt.Printf("Removing finalizers from namespace '%s'...\n", namespace) + + k8sURL := fmt.Sprintf("http://127.0.0.1:%d/api/v1/namespaces/%s/finalize", port, namespace) + req, err := retryablehttp.NewRequest("PUT", k8sURL, bytes.NewReader(namespaceUpdateBytes)) + if err != nil { + fmt.Println("Error creating the request to update the namespace:", err) + return err + } + req.Header.Set("Content-Type", "application/json") + + errMsg := "Error removing finalizer from namespace" + client := retryablehttp.NewClient() + client.Logger = nil + if resp, err := client.Do(req); err != nil { + return errors.Wrap(err, errMsg) + } else if resp.StatusCode != http.StatusOK { + return errors.New(errMsg) + } + + return nil + } + + err = k.exec.RunWithProxy(fun) + if err != nil { + return errors.Wrapf(err, "Cannot force-delete namespace '%s'", namespace) + } + + // Give it some more time to be deleted by K8s + time.Sleep(5 * time.Second) + + // Check again + if _, err := k.exec.RunProcessAndCaptureOutput("kubectl", "get", "namespace", namespace); err != nil { + fmt.Printf("Namespace '%s' terminated.\n", namespace) + return nil + } + + fmt.Printf("Force-deleting namespace '%s'...\n", namespace) + if err := k.exec.RunProcess("kubectl", "delete", "namespace", namespace, "--force", "--grace-period=0", "--ignore-not-found=true"); err != nil { + fmt.Println("Error deleting namespace:", err) + return err + } + + return nil +} + func (k Kubectl) WaitForDeployments(namespace string, selector string) error { output, err := k.exec.RunProcessAndCaptureOutput( "kubectl", "get", "deployments", "--namespace", namespace, "--selector", selector, "--output", "jsonpath={.items[*].metadata.name}") diff --git a/pkg/util/util.go b/pkg/util/util.go index a73856b..a098d4f 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -21,6 +21,7 @@ import ( "gopkg.in/yaml.v2" "io/ioutil" "math/rand" + "net" "os" "path" "path/filepath" @@ -203,3 +204,13 @@ func TruncateLeft(s string, maxLength int) string { } return s } + +func GetRandomPort() (int , error) { + listener, err := net.Listen("tcp", ":0") + defer listener.Close() + if err != nil { + return 0, err + } + + return listener.Addr().(*net.TCPAddr).Port, nil +}