From 136e5065497c079dce4a28fddfaf781457a2dc8b Mon Sep 17 00:00:00 2001 From: Max Dudin Date: Wed, 17 Jun 2020 16:14:55 +0300 Subject: [PATCH] CLOUD-535 Add cluster debug collector --- src/go/debug-collector/archive/archive.go | 58 +++++++ src/go/debug-collector/dumper/dumper.go | 202 ++++++++++++++++++++++ src/go/debug-collector/main.go | 33 ++++ 3 files changed, 293 insertions(+) create mode 100644 src/go/debug-collector/archive/archive.go create mode 100644 src/go/debug-collector/dumper/dumper.go create mode 100644 src/go/debug-collector/main.go diff --git a/src/go/debug-collector/archive/archive.go b/src/go/debug-collector/archive/archive.go new file mode 100644 index 00000000..9470edbb --- /dev/null +++ b/src/go/debug-collector/archive/archive.go @@ -0,0 +1,58 @@ +package archive + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "os" + "path/filepath" +) + +func Create(src string) error { + var buf bytes.Buffer + + zr := gzip.NewWriter(&buf) + tw := tar.NewWriter(zr) + + filepath.Walk(src, func(file string, fi os.FileInfo, err error) error { + header, err := tar.FileInfoHeader(fi, file) + if err != nil { + return err + } + header.Name = filepath.ToSlash(file) + + err = tw.WriteHeader(header) + if err != nil { + return err + } + + if !fi.IsDir() { + data, err := os.Open(file) + if err != nil { + return err + } + if _, err := io.Copy(tw, data); err != nil { + return err + } + } + return nil + }) + + if err := tw.Close(); err != nil { + return err + } + if err := zr.Close(); err != nil { + return err + } + + file, err := os.OpenFile("./cluster-dump.tar.gzip", os.O_CREATE|os.O_RDWR, os.FileMode(777)) + if err != nil { + return err + } + if _, err := io.Copy(file, &buf); err != nil { + return err + } + + return nil +} diff --git a/src/go/debug-collector/dumper/dumper.go b/src/go/debug-collector/dumper/dumper.go new file mode 100644 index 00000000..2a57bc17 --- /dev/null +++ b/src/go/debug-collector/dumper/dumper.go @@ -0,0 +1,202 @@ +package dumper + +import ( + "bytes" + "encoding/json" + "errors" + "log" + "os" + "os/exec" + + corev1 "k8s.io/api/core/v1" +) + +// Dumper struct is for dumping cluster +type Dumper struct { + cmd string + resources []string + location string + Errors map[string]string +} + +// New return new Dumper object +func New(location string) Dumper { + directory := "cluster-dump" + if len(location) > 0 { + directory = location + "/cluster-dump" + } + return Dumper{ + cmd: "kubectl", + resources: []string{ + "pods", + "replicasets", + "deployments", + "daemonsets", + "replicationcontrollers", + "events", + }, + location: directory, + Errors: make(map[string]string), + } +} + +type k8sPods struct { + Items []corev1.Pod `json:"items"` +} + +type namespaces struct { + Items []corev1.Namespace `json:"items"` +} + +// DumpCluster create dump of a cluster in Dumper.location +func (d *Dumper) DumpCluster() error { + output, err := d.runCmd("get", "namespaces", "-o", "json") + if err != nil { + return err + } + var nss namespaces + err = json.Unmarshal(output, &nss) + if err != nil { + return err + } + + for _, ns := range nss.Items { + err = os.MkdirAll(d.location+"/"+ns.Name, 0755) + if err != nil { + return err + } + + output, err = d.runCmd("get", "pods", "-o", "json", "--namespace", ns.Name) + if err != nil { + continue // runCmd already stored this error in Dumper.Errors + } + var pods k8sPods + err = json.Unmarshal(output, &pods) + if err != nil { + return err + } + + for _, pod := range pods.Items { + output, err = d.runCmd("logs", pod.Name, "--namespace", ns.Name, "--all-containers") + if err != nil { + continue // runCmd already stored this error in Dumper.Errors + } + err = os.MkdirAll(d.location+"/"+ns.Name+"/"+pod.Name, 0755) + if err != nil { + return err + } + + f, err := os.Create(d.location + "/" + ns.Name + "/" + pod.Name + "/logs.txt") + if err != nil { + return err + } + _, err = f.Write(output) + if err != nil { + return err + } + + } + + for _, resource := range d.resources { + err = d.getAndWriteToFile(resource, ns.Name) + if err != nil { + log.Println(err) + } + } + } + + err = d.getAndWriteToFile("nodes", "") + if err != nil { + log.Println(err) + } + + err = d.writeErrorsToFile() + if err != nil { + log.Println(err) + } + + return nil +} + +// runCmd run command (Dumper.cmd) with given args, return it output and save all errors in Dumper.errors +func (d *Dumper) runCmd(args ...string) ([]byte, error) { + var outb, errb bytes.Buffer + cmd := exec.Command(d.cmd, args...) + cmd.Stdout = &outb + cmd.Stderr = &errb + err := cmd.Run() + if err != nil { + d.saveCommandError(err.Error()+" "+outb.String(), args...) + return outb.Bytes(), err + } + if len(errb.String()) > 0 { + d.saveCommandError(errb.String()+" "+outb.String(), args...) + return outb.Bytes(), err + } + + return outb.Bytes(), nil +} + +func (d *Dumper) getAndWriteToFile(name, namespace string) error { + location := d.location + args := []string{"get", name, "-o", "yaml"} + if len(namespace) > 0 { + args = append(args, "--namespace", namespace) + location = d.location + "/" + namespace + } + output, err := d.runCmd(args...) + if err != nil { + return nil // runCmd already stored this error in Dumper.Errors + } + + f, err := os.Create(location + "/" + name + ".yaml") + if err != nil { + return err + } + _, err = f.Write(output) + if err != nil { + return err + } + + return nil +} + +func (d *Dumper) saveCommandError(err string, args ...string) { + command := d.cmd + for _, arg := range args { + command += " " + arg + } + + d.Errors[command] = err +} + +func (d *Dumper) writeErrorsToFile() error { + var errStr string + for cmd, errS := range d.Errors { + errStr += cmd + ": " + errS + "\n" + } + f, err := os.Create(d.location + "/errors.txt") + if err != nil { + return err + } + _, err = f.WriteString(errStr) + if err != nil { + return err + } + + return nil +} + +// DeleteDumpDir delete Dumper.location +func (d *Dumper) DeleteDumpDir() error { + if d.location == "/" { + return errors.New("don't do this, please") // just for being sure + } + + return os.RemoveAll(d.location) +} + +// GetLocation return Dumper.location +func (d *Dumper) GetLocation() string { + return d.location +} diff --git a/src/go/debug-collector/main.go b/src/go/debug-collector/main.go new file mode 100644 index 00000000..0d4ededb --- /dev/null +++ b/src/go/debug-collector/main.go @@ -0,0 +1,33 @@ +package main + +import ( + "log" + "os" + + "github.com/percona/percona-toolkit/src/go/debug-collector/archive" + "github.com/percona/percona-toolkit/src/go/debug-collector/dumper" +) + +func main() { + locations := "" + if len(os.Args) > 1 { + locations = os.Args[1] + } + d := dumper.New(locations) + log.Println("Start dump cluster") + err := d.DumpCluster() + if err != nil { + log.Println(err) + os.Exit(1) + } + + archive.Create(d.GetLocation()) + + err = d.DeleteDumpDir() + if err != nil { + log.Println(err) + os.Exit(1) + } + + log.Println("Cluster dump ready") +}