From faffa70867167a46b54056c9b2b298bcc668edcc Mon Sep 17 00:00:00 2001 From: Max Dudin Date: Wed, 1 Jul 2020 10:53:28 +0300 Subject: [PATCH] PT-1865 Remove archive package --- src/go/debug-collector/archive/archive.go | 35 ------ src/go/debug-collector/dumper/dumper.go | 144 ++++++++++++++++------ src/go/debug-collector/main.go | 24 ++-- 3 files changed, 117 insertions(+), 86 deletions(-) delete mode 100644 src/go/debug-collector/archive/archive.go diff --git a/src/go/debug-collector/archive/archive.go b/src/go/debug-collector/archive/archive.go deleted file mode 100644 index 77da0090..00000000 --- a/src/go/debug-collector/archive/archive.go +++ /dev/null @@ -1,35 +0,0 @@ -package archive - -import ( - "archive/tar" - "compress/gzip" - "os" - - "github.com/pkg/errors" -) - -func TarWrite(path string, data map[string][]byte) error { - tarFile, err := os.Create(path + ".tar.gz") - if err != nil { - return errors.Wrap(err, "create tar file") - } - defer tarFile.Close() - zr := gzip.NewWriter(tarFile) - tw := tar.NewWriter(zr) - defer zr.Close() - defer tw.Close() - for name, content := range data { - hdr := &tar.Header{ - Name: name, - Mode: 0600, - Size: int64(len(content)), - } - if err := tw.WriteHeader(hdr); err != nil { - return errors.Wrap(err, "write header") - } - if _, err := tw.Write(content); err != nil { - return errors.Wrap(err, "write content") - } - } - return nil -} diff --git a/src/go/debug-collector/dumper/dumper.go b/src/go/debug-collector/dumper/dumper.go index 208a21f7..bd2b1c0d 100644 --- a/src/go/debug-collector/dumper/dumper.go +++ b/src/go/debug-collector/dumper/dumper.go @@ -1,9 +1,12 @@ package dumper import ( + "archive/tar" "bytes" + "compress/gzip" "encoding/json" "log" + "os" "os/exec" "strings" @@ -15,30 +18,53 @@ import ( type Dumper struct { cmd string resources []string + namespace string location string Errors map[string]string - Files map[string][]byte + mode int64 } // New return new Dumper object -func New(location string) Dumper { +func New(location, namespace, resource string) Dumper { directory := "cluster-dump" if len(location) > 0 { directory = location + "/cluster-dump" } + resources := []string{ + "pods", + "replicasets", + "deployments", + "statefulsets", + "replicationcontrollers", + "events", + "configmaps", + "secrets", + "cronjobs", + "jobs", + "podsecuritypolicies", + "poddisruptionbudgets", + "perconaxtradbbackups", + "perconaxtradbclusterbackups", + "perconaxtradbclusterrestores", + "perconaxtradbclusters", + "clusterrolebindings", + "clusterroles", + "rolebindings", + "roles", + "storageclasses", + "persistentvolumeclaims", + "persistentvolumes", + } + if len(resource) > 0 { + resources = append(resources, resource) + } return Dumper{ - cmd: "kubectl", - resources: []string{ - "pods", - "replicasets", - "deployments", - "daemonsets", - "replicationcontrollers", - "events", - }, - location: directory, - Errors: make(map[string]string), - Files: make(map[string][]byte), + cmd: "kubectl", + resources: resources, + location: directory, + Errors: make(map[string]string), + mode: int64(0777), + namespace: namespace, } } @@ -52,53 +78,80 @@ type namespaces struct { // DumpCluster create dump of a cluster in Dumper.location func (d *Dumper) DumpCluster() error { - output, err := d.runCmd("get", "namespaces", "-o", "json") + tarFile, err := os.Create(d.location + ".tar.gz") if err != nil { - return errors.Wrap(err, "get namespaces") + return errors.Wrap(err, "create tar file") } + defer tarFile.Close() + zr := gzip.NewWriter(tarFile) + tw := tar.NewWriter(zr) + defer zr.Close() + defer tw.Close() + var nss namespaces - err = json.Unmarshal(output, &nss) - if err != nil { - return errors.Wrap(err, "unmarshal namespaces") + + if len(d.namespace) > 0 { + ns := corev1.Namespace{} + ns.Name = d.namespace + nss.Items = append(nss.Items, ns) + } else { + output, err := d.runCmd("get", "namespaces", "-o", "json") + if err != nil { + return errors.Wrap(err, "get namespaces") + } + + err = json.Unmarshal(output, &nss) + if err != nil { + return errors.Wrap(err, "unmarshal namespaces") + } } for _, ns := range nss.Items { - output, err = d.runCmd("get", "pods", "-o", "json", "--namespace", ns.Name) + output, err := d.runCmd("get", "pods", "-o", "json", "--namespace", ns.Name) if err != nil { continue // runCmd already stored this error in Dumper.Errors } var pods k8sPods err = json.Unmarshal(output, &pods) if err != nil { + d.saveCommandError(err.Error(), "unmarshal pods from namespace", ns.Name) log.Println(errors.Wrap(err, "unmarshal pods")) } for _, pod := range pods.Items { + location := d.location + "/" + ns.Name + "/" + pod.Name + "/logs.txt" output, err = d.runCmd("logs", pod.Name, "--namespace", ns.Name, "--all-containers") if err != nil { - continue // runCmd already stored this error in Dumper.Errors + err = createArchive(location, d.mode, []byte(err.Error()), tw) + if err != nil { + log.Printf("create archive with err: %v", err) + } + continue + } + err = createArchive(location, d.mode, output, tw) + if err != nil { + log.Printf("create archive for pod %s: %v", pod.Name, err) } - d.Files[d.location+"/"+ns.Name+"/"+pod.Name+"/logs.txt"] = output } for _, resource := range d.resources { - err = d.getResource(resource, ns.Name) + err = d.getResource(resource, ns.Name, tw) if err != nil { log.Println(errors.Wrapf(err, "get %s resource", resource)) } } + + err = d.writeErrorsToFile(tw) + if err != nil { + log.Println(errors.Wrap(err, "write errors")) + } } - err = d.getResource("nodes", "") + err = d.getResource("nodes", "", tw) if err != nil { log.Println(errors.Wrapf(err, "get nodes")) } - err = d.writeErrorsToFile() - if err != nil { - log.Println(errors.Wrap(err, "write errors")) - } - return nil } @@ -121,21 +174,20 @@ func (d *Dumper) runCmd(args ...string) ([]byte, error) { return outb.Bytes(), nil } -func (d *Dumper) getResource(name, namespace string) error { +func (d *Dumper) getResource(name, namespace string, tw *tar.Writer) error { location := d.location args := []string{"get", name, "-o", "yaml"} if len(namespace) > 0 { args = append(args, "--namespace", namespace) location = d.location + "/" + namespace } + location += "/" + name + ".yaml" output, err := d.runCmd(args...) if err != nil { - return nil // runCmd already stored this error in Dumper.Errors + return createArchive(location, d.mode, []byte(err.Error()), tw) } - d.Files[location+"/"+name+".yaml"] = output - - return nil + return createArchive(location, d.mode, output, tw) } func (d *Dumper) saveCommandError(err string, args ...string) { @@ -144,17 +196,27 @@ func (d *Dumper) saveCommandError(err string, args ...string) { d.Errors[command] = err } -func (d *Dumper) writeErrorsToFile() error { +func (d *Dumper) writeErrorsToFile(tw *tar.Writer) error { var errStr string for cmd, errS := range d.Errors { errStr += cmd + ": " + errS + "\n" } - d.Files[d.location+"/errors.txt"] = []byte(errStr) + + return createArchive(d.location+"/errors.txt", d.mode, []byte(errStr), tw) +} + +func createArchive(location string, mode int64, content []byte, tw *tar.Writer) error { + hdr := &tar.Header{ + Name: location, + Mode: mode, + Size: int64(len(content)), + } + if err := tw.WriteHeader(hdr); err != nil { + return errors.Wrap(err, "write header") + } + if _, err := tw.Write(content); err != nil { + return errors.Wrapf(err, "write content to %s", location) + } return nil } - -// GetLocation return Dumper.location -func (d *Dumper) GetLocation() string { - return d.location -} diff --git a/src/go/debug-collector/main.go b/src/go/debug-collector/main.go index 5f5b75e8..d849350d 100644 --- a/src/go/debug-collector/main.go +++ b/src/go/debug-collector/main.go @@ -1,19 +1,29 @@ package main import ( + "flag" "log" "os" - "github.com/percona/percona-toolkit/src/go/debug-collector/archive" "github.com/percona/percona-toolkit/src/go/debug-collector/dumper" ) func main() { locations := "" - if len(os.Args) > 1 { - locations = os.Args[1] + namespace := "" + resource := "" + clusterName := "" + + flag.StringVar(&namespace, "namespace", "", "Namespace for dumping. If empty will dump all namespaces") + flag.StringVar(&resource, "resource", "pxc", "Resource name. Default value - 'pxc'") + flag.StringVar(&clusterName, "cluster", "", "Cluster name") + flag.Parse() + + if len(clusterName) > 0 { + resource = resource + "/" + clusterName } - d := dumper.New(locations) + + d := dumper.New(locations, namespace, resource) log.Println("Start dump cluster") err := d.DumpCluster() if err != nil { @@ -21,11 +31,5 @@ func main() { os.Exit(1) } - err = archive.TarWrite(d.GetLocation(), d.Files) - if err != nil { - log.Println(err) - os.Exit(1) - } - log.Println("Cluster dump ready") }