From c2419ba10a66650c20d4032c5cc10004317a4bf0 Mon Sep 17 00:00:00 2001 From: Carlos Salguero Date: Wed, 11 Jan 2017 15:01:33 -0300 Subject: [PATCH] Implemented version check Updated readme --- src/go/Makefile | 3 +- src/go/lib/config/config.go | 150 +++++++++++++++++ src/go/lib/config/config_test.go | 154 +++++++++++++++++ src/go/lib/util/util.go | 20 +++ src/go/lib/versioncheck/version_check.go | 90 ++++++++++ src/go/lib/versioncheck/version_check_test.go | 58 +++++++ src/go/pt-mongodb-query-profiler/README.md | 44 +++++ src/go/pt-mongodb-query-profiler/main.go | 89 ++++++---- src/go/pt-mongodb-summary/main.go | 155 ++++++++---------- src/go/pt-mongodb-summary/main_test.go | 26 +-- .../pt-mongodb-summary/{ => oplog}/oplog.go | 2 +- src/go/pt-mongodb-summary/oplog/oplog_test.go | 1 + src/go/tests/lib/sample-config1.conf | 9 + src/go/tests/lib/sample-config2.conf | 10 ++ 14 files changed, 681 insertions(+), 130 deletions(-) create mode 100644 src/go/lib/config/config.go create mode 100644 src/go/lib/config/config_test.go create mode 100644 src/go/lib/util/util.go create mode 100644 src/go/lib/versioncheck/version_check.go create mode 100644 src/go/lib/versioncheck/version_check_test.go create mode 100644 src/go/pt-mongodb-query-profiler/README.md rename src/go/pt-mongodb-summary/{ => oplog}/oplog.go (99%) create mode 100644 src/go/pt-mongodb-summary/oplog/oplog_test.go create mode 100644 src/go/tests/lib/sample-config1.conf create mode 100644 src/go/tests/lib/sample-config2.conf diff --git a/src/go/Makefile b/src/go/Makefile index 951211cd..38a874c3 100644 --- a/src/go/Makefile +++ b/src/go/Makefile @@ -2,10 +2,11 @@ GO := go pkgs = $(shell $(GO) list ./... | grep -v /vendor/) VERSION=$(shell git describe --tags) BUILD=$(shell date +%FT%T%z) +GOVERSION=$(shell go version | cut --delimiter=" " -f3) PREFIX=$(shell pwd) BIN_DIR=$(shell git rev-parse --show-toplevel)/bin -LDFLAGS="-X main.Version=${VERSION} -X main.Build=${BUILD}" +LDFLAGS="-X main.Version=${VERSION} -X main.Build=${BUILD} -X main.GoVersion=${GOVERSION}" build-all: diff --git a/src/go/lib/config/config.go b/src/go/lib/config/config.go new file mode 100644 index 00000000..0c35c1c0 --- /dev/null +++ b/src/go/lib/config/config.go @@ -0,0 +1,150 @@ +package config + +import ( + "bufio" + "os" + "os/user" + "strconv" + "strings" +) + +type Config struct { + options map[string]interface{} +} + +func (c *Config) GetString(key string) string { + if val, ok := c.options[key]; ok { + if v, ok := val.(string); ok { + return v + } + } + return "" +} + +func (c *Config) GetInt64(key string) int64 { + if val, ok := c.options[key]; ok { + if v, ok := val.(int64); ok { + return v + } + } + return 0 +} + +func (c *Config) GetFloat64(key string) float64 { + if val, ok := c.options[key]; ok { + if v, ok := val.(float64); ok { + return v + } + } + return 0 +} + +func (c *Config) GetBool(key string) bool { + if val, ok := c.options[key]; ok { + if v, ok := val.(bool); ok { + return v + } + } + return false +} + +func (c *Config) HasKey(key string) bool { + _, ok := c.options[key] + return ok +} + +func DefaultConfigFiles(toolName string) ([]string, error) { + user, err := user.Current() + if err != nil { + return nil, err + } + + files := []string{ + "/etc/percona-toolkit/percona-toolkit.conf", + "/etc/percona-toolkit/${TOOLNAME}.conf", + "${HOME}/.percona-toolkit.conf", + "${HOME}/.${TOOLNAME}.conf", + } + + for i := 0; i < len(files); i++ { + files[i] = strings.Replace(files[i], "${TOOLNAME}", toolName, -1) + files[i] = strings.Replace(files[i], "${HOME}", user.HomeDir, -1) + } + + return files, nil +} + +func DefaultConfig(toolname string) *Config { + + files, _ := DefaultConfigFiles(toolname) + return NewConfig(files...) + +} + +func NewConfig(files ...string) *Config { + config := &Config{ + options: make(map[string]interface{}), + } + for _, filename := range files { + if _, err := os.Stat(filename); err == nil { + read(filename, config.options) + } + } + return config +} + +func read(filename string, opts map[string]interface{}) error { + + f, err := os.Open(filename) + if err != nil { + return err + } + + scanner := bufio.NewScanner(f) + + for scanner.Scan() { + line := scanner.Text() + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + m := strings.SplitN(scanner.Text(), "=", 2) + key := strings.TrimSpace(m[0]) + + if len(m) == 1 { + opts[key] = true + continue + } + + val := strings.TrimSpace(m[1]) + lcval := strings.ToLower(val) + + if lcval == "true" || lcval == "yes" { + opts[key] = true + continue + } + if lcval == "false" || lcval == "no" { + opts[key] = false + continue + } + + f, err := strconv.ParseFloat(val, 64) + if err != nil { + opts[key] = strings.TrimSpace(val) // string + continue + } + + if f == float64(int64(f)) { + opts[key] = int64(f) //int64 + continue + } + + opts[key] = f // float64 + } + + if err := scanner.Err(); err != nil { + return err + } + + return nil +} diff --git a/src/go/lib/config/config_test.go b/src/go/lib/config/config_test.go new file mode 100644 index 00000000..83c5e029 --- /dev/null +++ b/src/go/lib/config/config_test.go @@ -0,0 +1,154 @@ +package config + +import ( + "fmt" + "os/user" + "path" + "reflect" + "testing" + + "github.com/percona/percona-toolkit/src/go/lib/util" +) + +func TestReadConfig(t *testing.T) { + + rootPath, err := util.RootPath() + if err != nil { + t.Errorf("cannot get root path: %s", err) + } + file := path.Join(rootPath, "src/go/tests/lib/sample-config1.conf") + + conf := NewConfig(file) + + keys := []string{"no-version-check", "trueboolvar", "yesboolvar", "noboolvar", "falseboolvar", "intvar", "floatvar", "stringvar"} + for _, key := range keys { + if !conf.HasKey(key) { + t.Errorf("missing %s key", key) + } + } + + // no-version-check + if conf.GetBool("no-version-check") != true { + t.Error("no-version-check should be enabled") + } + + // trueboolvar=true + if conf.GetBool("trueboolvar") != true { + t.Error("trueboolvar should be true") + } + + // yesboolvar=yes + if conf.GetBool("yesboolvar") != true { + t.Error("yesboolvar should be true") + } + + // falseboolvar=false + if conf.GetBool("falseboolvar") != false { + t.Error("trueboolvar should be false") + } + + // noboolvar=no + if conf.GetBool("noboolvar") != false { + t.Error("yesboolvar should be false") + } + + // intvar=1 + if got := conf.GetInt64("intvar"); got != 1 { + t.Errorf("intvar should be 1, got %d", got) + } + + // floatvar=2.3 + if got := conf.GetFloat64("floatvar"); got != 2.3 { + t.Errorf("floatvar should be 2.3, got %f", got) + } + + // stringvar=some string var having = and # + if got := conf.GetString("stringvar"); got != "some string var having = and #" { + t.Errorf("string var incorect value; got %s", got) + } +} + +func TestOverrideConfig(t *testing.T) { + + rootPath, err := util.RootPath() + if err != nil { + t.Errorf("cannot get root path: %s", err) + } + file1 := path.Join(rootPath, "src/go/tests/lib/sample-config1.conf") + file2 := path.Join(rootPath, "src/go/tests/lib/sample-config2.conf") + + conf := NewConfig(file1, file2) + + keys := []string{"no-version-check", "trueboolvar", "yesboolvar", "noboolvar", "falseboolvar", "intvar", "floatvar", "stringvar"} + for _, key := range keys { + if !conf.HasKey(key) { + t.Errorf("missing %s key", key) + } + } + + // no-version-check. This option is missing in the 2nd file. + // It should remain unchanged + if conf.GetBool("no-version-check") != true { + t.Error("no-version-check should be enabled") + } + + if conf.GetBool("trueboolvar") == true { + t.Error("trueboolvar should be false") + } + + if conf.GetBool("yesboolvar") == true { + t.Error("yesboolvar should be false") + } + + if conf.GetBool("falseboolvar") == false { + t.Error("trueboolvar should be true") + } + + if conf.GetBool("noboolvar") == false { + t.Error("yesboolvar should be true") + } + + if got := conf.GetInt64("intvar"); got != 4 { + t.Errorf("intvar should be 4, got %d", got) + } + + if got := conf.GetFloat64("floatvar"); got != 5.6 { + t.Errorf("floatvar should be 5.6, got %f", got) + } + + if got := conf.GetString("stringvar"); got != "some other string" { + t.Errorf("string var incorect value; got %s", got) + } + + // This exists only in file2 + if got := conf.GetString("newstring"); got != "a new string" { + t.Errorf("string var incorect value; got %s", got) + } + + if got := conf.GetInt64("anotherint"); got != 8 { + t.Errorf("intvar should be 8, got %d", got) + } +} + +func TestDefaultFiles(t *testing.T) { + + user, _ := user.Current() + toolname := "pt-testing" + + want := []string{ + "/etc/percona-toolkit/percona-toolkit.conf", + fmt.Sprintf("/etc/percona-toolkit/%s.conf", toolname), + fmt.Sprintf("%s/.percona-toolkit.conf", user.HomeDir), + fmt.Sprintf("%s/.%s.conf", user.HomeDir, toolname), + } + + got, err := DefaultConfigFiles(toolname) + if err != nil { + t.Errorf("cannot get default config files list: %s", err) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("got %#v\nwant: %#v\n", got, want) + } + +} diff --git a/src/go/lib/util/util.go b/src/go/lib/util/util.go new file mode 100644 index 00000000..a99af609 --- /dev/null +++ b/src/go/lib/util/util.go @@ -0,0 +1,20 @@ +package util + +import ( + "encoding/json" + "os/exec" + "strings" +) + +func RootPath() (string, error) { + out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() + if err != nil { + return "", err + } + return strings.TrimSpace(string(out)), nil +} + +func Pretty(value interface{}) string { + bytes, _ := json.MarshalIndent(value, "", " ") + return string(bytes) +} diff --git a/src/go/lib/versioncheck/version_check.go b/src/go/lib/versioncheck/version_check.go new file mode 100644 index 00000000..10bbc1d3 --- /dev/null +++ b/src/go/lib/versioncheck/version_check.go @@ -0,0 +1,90 @@ +package versioncheck + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/kylelemons/godebug/pretty" + uuid "github.com/satori/go.uuid" + log "github.com/sirupsen/logrus" +) + +const ( + PERCONA_TOOLKIT = "Percona::Toolkit" + DEFAULT_TIMEOUT = 3 * time.Second + DEFAULT_URL = "https://v.percona.com/" + + URL_ENV_VAR = "PERCONA_VERSION_CHECK_URL" + TIMEOUT_ENV_VAR = "PERCONA_VERSION_CHECK_TIMEOUT" +) + +type Advice struct { + Hash string + ToolName string + Advice string +} + +func CheckUpdates(toolName, version string) (string, error) { + url := DEFAULT_URL + timeout := DEFAULT_TIMEOUT + + log.Info("Checking for updates") + if envURL := os.Getenv(URL_ENV_VAR); envURL != "" { + url = envURL + log.Infof("Using %s env var", URL_ENV_VAR) + } + + if envTimeout := os.Getenv(TIMEOUT_ENV_VAR); envTimeout != "" { + i, err := strconv.Atoi(envTimeout) + if err == nil && i > 0 { + log.Infof("Using time out from %s env var", TIMEOUT_ENV_VAR) + timeout = time.Millisecond * time.Duration(i) + } + } + + log.Infof("Contacting version check API at %s. Timeout set to %v", url, timeout) + return checkUpdates(url, timeout, toolName, version) +} + +func checkUpdates(url string, timeout time.Duration, toolName, version string) (string, error) { + + client := &http.Client{ + Timeout: timeout, + } + payload := fmt.Sprintf("%x;%s;%s", uuid.NewV2(uuid.DomainOrg).String(), PERCONA_TOOLKIT, version) + req, err := http.NewRequest("POST", url, strings.NewReader(payload)) + if err != nil { + return "", err + } + req.Header.Add("Accept", "application/json") + req.Header.Add("X-Percona-Toolkit-Tool", toolName) + resp, err := client.Do(req) + if err != nil { + return "", err + } + + log.Debug(pretty.Sprint(resp)) + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + advices := []Advice{} + err = json.Unmarshal(body, &advices) + if err != nil { + return "", err + } + + for _, advice := range advices { + if advice.ToolName == PERCONA_TOOLKIT { + return advice.Advice, nil + } + } + + return "", nil +} diff --git a/src/go/lib/versioncheck/version_check_test.go b/src/go/lib/versioncheck/version_check_test.go new file mode 100644 index 00000000..aa955c35 --- /dev/null +++ b/src/go/lib/versioncheck/version_check_test.go @@ -0,0 +1,58 @@ +package versioncheck + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestCheckUpdates(t *testing.T) { + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := ioutil.ReadAll(r.Body) + m := strings.Split(string(body), ";") + + advices := []Advice{ + Advice{ + Hash: m[0], + ToolName: m[1], + Advice: "There is a new version", + }, + } + + buf, _ := json.Marshal(advices) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, string(buf)) + })) + defer ts.Close() + + msg, err := CheckUpdates(ts.URL, "pt-test", "2.2.18") + if err != nil { + t.Errorf("error while checking %s", err) + } + if msg == "" { + t.Error("got empty response") + } + +} + +func TestEmptyResponse(t *testing.T) { + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "") + })) + defer ts.Close() + + msg, err := CheckUpdates(ts.URL, "pt-test", "2.2.18") + if err == nil { + t.Error("response should return error due to empty body") + } + if msg != "" { + t.Error("response should return error due to empty body") + } + +} diff --git a/src/go/pt-mongodb-query-profiler/README.md b/src/go/pt-mongodb-query-profiler/README.md new file mode 100644 index 00000000..094f3079 --- /dev/null +++ b/src/go/pt-mongodb-query-profiler/README.md @@ -0,0 +1,44 @@ +#pt-mongodb-query-digest + +This program reports query usage statistics by aggregating queries from MongoDB query profiler. +The queries are the result of running: +```javascript +db.getSiblingDB("samples").system.profile.find({"op":{"$nin":["getmore", "delete"]}}); +``` +and then, the results are grouped by fingerprint and namespace (database.collection). + +The fingerprint is calculated as the **sorted list** of the keys in the document. The max depth level is 10. +The last step is sorting the results. The default sort order is by ascending query count. + +##Sample output +``` +# Query 2: 0.00 QPS, ID 1a6443c2db9661f3aad8edb6b877e45d +# Ratio 1.00 (docs scanned/returned) +# Time range: 2017-01-11 12:58:26.519 -0300 ART to 2017-01-11 12:58:26.686 -0300 ART +# Attribute pct total min max avg 95% stddev median +# ================== === ======== ======== ======== ======== ======== ======= ======== +# Count (docs) 36 +# Exec Time ms 0 0 0 0 0 0 0 0 +# Docs Scanned 0 148.00 0.00 74.00 4.11 74.00 16.95 0.00 +# Docs Returned 2 148.00 0.00 74.00 4.11 74.00 16.95 0.00 +# Bytes recv 0 2.11M 215.00 1.05M 58.48K 1.05M 240.22K 215.00 +# String: +# Namespaces samples.col1 +# Fingerprint $gte,$lt,$meta,$sortKey,filter,find,projection,shardVersion,sort,user_id,user_id +``` + +##Command line parameters + +|Short|Long|Help| +|-----|----|----| +|-?|--help|Show help| +|-a|--authenticationDatabase|database used to establish credentials and privileges with a MongoDB server admin| +|-c|--no-version-check|Don't check for updates| +|-d|--database|database to profile| +|-l|--log-level|Log level:, panic, fatal, error, warn, info, debug error| +|-n|--limit|show the first n queries| +|-o|--order-by|comma separated list of order by fields (max values): `count`, `ratio`, `query-time`, `docs-scanned`, `docs-returned`.
A `-` in front of the field name denotes reverse order.
Example:`--order-by="count,-ratio"`).| +|-p|--password[=password]|Password (optional). If it is not specified it will be asked| +|-u|--user|Username| +|-v|--version|Show version & exit| + diff --git a/src/go/pt-mongodb-query-profiler/main.go b/src/go/pt-mongodb-query-profiler/main.go index 429fe21d..7b09c0db 100644 --- a/src/go/pt-mongodb-query-profiler/main.go +++ b/src/go/pt-mongodb-query-profiler/main.go @@ -4,7 +4,6 @@ import ( "crypto/md5" "fmt" "html/template" - "log" "os" "sort" "strings" @@ -13,14 +12,23 @@ import ( "github.com/howeyc/gopass" "github.com/montanaflynn/stats" "github.com/pborman/getopt" + "github.com/percona/percona-toolkit/src/go/lib/config" + "github.com/percona/percona-toolkit/src/go/lib/versioncheck" "github.com/percona/percona-toolkit/src/go/mongolib/proto" + log "github.com/sirupsen/logrus" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" ) +const ( + TOOLNAME = "pt-mongodb-query-digest" + MAX_DEPTH_LEVEL = 10 +) + var ( - Version string - Build string + Version string + Build string + GoVersion string ) type iter interface { @@ -33,22 +41,20 @@ type iter interface { } type options struct { - AuthDB string - Database string - Debug bool - Help bool - Host string - Limit int - OrderBy []string - Password string - User string - Version bool + AuthDB string + Database string + Debug bool + Help bool + Host string + Limit int + LogLevel string + NoVersionCheck bool + OrderBy []string + Password string + User string + Version bool } -const ( - MAX_DEPTH_LEVEL = 10 -) - type statsArray []stat func (a statsArray) Len() int { return len(a) } @@ -95,19 +101,20 @@ type statistics struct { } type queryInfo struct { - Rank int - ID string Count int - Ratio float64 - QPS float64 Fingerprint string - Namespace string - Scanned statistics - Returned statistics - QueryTime statistics - ResponseLength statistics FirstSeen time.Time + ID string LastSeen time.Time + Namespace string + NoVersionCheck bool + QPS float64 + QueryTime statistics + Rank int + Ratio float64 + ResponseLength statistics + Returned statistics + Scanned statistics } func main() { @@ -122,13 +129,31 @@ func main() { return } + logLevel, err := log.ParseLevel(opts.LogLevel) + if err != nil { + fmt.Printf("cannot set log level: %s", err.Error()) + } + log.SetLevel(logLevel) + if opts.Version { fmt.Println("pt-mongodb-summary") fmt.Printf("Version %s\n", Version) - fmt.Printf("Build: %s\n", Build) + fmt.Printf("Build: %s using %s\n", Build, GoVersion) return } + conf := config.DefaultConfig(TOOLNAME) + if !conf.GetBool("no-version-check") && !opts.NoVersionCheck { + advice, err := versioncheck.CheckUpdates(TOOLNAME, Version) + if err != nil { + log.Infof("cannot check version updates: %s", err.Error()) + } else { + if advice != "" { + log.Infof(advice) + } + } + } + di := getDialInfo(opts) if di.Database == "" { log.Printf("must indicate a database") @@ -379,18 +404,20 @@ func getData(i iter) []stat { } func getOptions() (*options, error) { - opts := &options{Host: "localhost:27017"} + opts := &options{Host: "localhost:27017", LogLevel: "error", OrderBy: []string{"count"}} getopt.BoolVarLong(&opts.Help, "help", '?', "Show help") - getopt.BoolVarLong(&opts.Version, "version", 'v', "", "show version & exit") + getopt.BoolVarLong(&opts.Version, "version", 'v', "show version & exit") + getopt.BoolVarLong(&opts.NoVersionCheck, "no-version-check", 'c', "Don't check for updates") - getopt.IntVarLong(&opts.Limit, "limit", 'l', "show the first n queries") + getopt.IntVarLong(&opts.Limit, "limit", 'n', "show the first n queries") getopt.ListVarLong(&opts.OrderBy, "order-by", 'o', "comma separated list of order by fields (max values): count,ratio,query-time,docs-scanned,docs-returned. - in front of the field name denotes reverse order.") getopt.StringVarLong(&opts.AuthDB, "authenticationDatabase", 'a', "admin", "database used to establish credentials and privileges with a MongoDB server") getopt.StringVarLong(&opts.Database, "database", 'd', "", "database to profile") + getopt.StringVarLong(&opts.LogLevel, "log-level", 'l', "error", "Log level:, panic, fatal, error, warn, info, debug") getopt.StringVarLong(&opts.Password, "password", 'p', "", "password").SetOptional() - getopt.StringVarLong(&opts.User, "user", 'u', "", "username") + getopt.StringVarLong(&opts.User, "user", 'u', "username") getopt.SetParameters("host[:port][/database]") diff --git a/src/go/pt-mongodb-summary/main.go b/src/go/pt-mongodb-summary/main.go index dc7be84a..96ce76b3 100644 --- a/src/go/pt-mongodb-summary/main.go +++ b/src/go/pt-mongodb-summary/main.go @@ -3,27 +3,52 @@ package main import ( "fmt" "html/template" - "log" "os" "strings" "time" "github.com/howeyc/gopass" "github.com/pborman/getopt" + "github.com/percona/percona-toolkit/src/go/lib/config" + "github.com/percona/percona-toolkit/src/go/lib/util" + "github.com/percona/percona-toolkit/src/go/lib/versioncheck" "github.com/percona/percona-toolkit/src/go/mongolib/proto" + "github.com/percona/percona-toolkit/src/go/pt-mongodb-summary/oplog" "github.com/percona/percona-toolkit/src/go/pt-mongodb-summary/templates" "github.com/percona/pmgo" "github.com/pkg/errors" "github.com/shirou/gopsutil/process" + log "github.com/sirupsen/logrus" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" ) -var ( - Version string - Build string +const ( + TOOLNAME = "pt-mongodb-summary" ) +var ( + Version string = "2.2.19" + Build string = "01-01-1980" + GoVersion string = "1.8" +) + +type TimedStats struct { + Min int64 + Max int64 + Total int64 + Avg int64 +} + +type opCounters struct { + Insert TimedStats + Query TimedStats + Update TimedStats + Delete TimedStats + GetMore TimedStats + Command TimedStats + SampleRate time.Duration +} type hostInfo struct { ThisHostID int Hostname string @@ -59,23 +84,6 @@ type security struct { SSL string } -type timedStats struct { - Min int64 - Max int64 - Total int64 - Avg int64 -} - -type opCounters struct { - Insert timedStats - Query timedStats - Update timedStats - Delete timedStats - GetMore timedStats - Command timedStats - SampleRate time.Duration -} - type databases struct { Databases []struct { Name string `bson:"name"` @@ -102,23 +110,26 @@ type clusterwideInfo struct { } type options struct { - Host string - User string - Password string - AuthDB string - Debug bool - Version bool + Host string + User string + Password string + AuthDB string + LogLevel string + Version bool + NoVersionCheck bool } func main() { - opts := options{Host: "localhost:27017"} + opts := options{Host: "localhost:27017", LogLevel: "error"} help := getopt.BoolLong("help", '?', "Show help") - getopt.BoolVarLong(&opts.Version, "version", 'v', "", "show version & exit") + getopt.BoolVarLong(&opts.Version, "version", 'v', "", "Show version & exit") + getopt.BoolVarLong(&opts.NoVersionCheck, "no-version-check", 'c', "", "Don't check for updates") - getopt.StringVarLong(&opts.User, "user", 'u', "", "username") - getopt.StringVarLong(&opts.Password, "password", 'p', "", "password").SetOptional() - getopt.StringVarLong(&opts.AuthDB, "authenticationDatabase", 'a', "admin", "database used to establish credentials and privileges with a MongoDB server") + getopt.StringVarLong(&opts.User, "user", 'u', "", "User name") + getopt.StringVarLong(&opts.Password, "password", 'p', "", "Password").SetOptional() + getopt.StringVarLong(&opts.AuthDB, "authenticationDatabase", 'a', "admin", "Database used to establish credentials and privileges with a MongoDB server") + getopt.StringVarLong(&opts.LogLevel, "log-level", 'l', "error", "Log level:, panic, fatal, error, warn, info, debug") getopt.SetParameters("host[:port]") getopt.Parse() @@ -127,6 +138,13 @@ func main() { return } + logLevel, err := log.ParseLevel(opts.LogLevel) + if err != nil { + fmt.Printf("cannot set log level: %s", err.Error()) + } + + log.SetLevel(logLevel) + args := getopt.Args() // positional arg if len(args) > 0 { opts.Host = args[0] @@ -135,10 +153,22 @@ func main() { if opts.Version { fmt.Println("pt-mongodb-summary") fmt.Printf("Version %s\n", Version) - fmt.Printf("Build: %s\n", Build) + fmt.Printf("Build: %s using %s\n", Build, GoVersion) return } + conf := config.DefaultConfig(TOOLNAME) + if !conf.GetBool("no-version-check") && !opts.NoVersionCheck { + advice, err := versioncheck.CheckUpdates(TOOLNAME, Version) + if err != nil { + log.Infof("cannot check version updates: %s", err.Error()) + } else { + if advice != "" { + log.Infof(advice) + } + } + } + if getopt.IsSet("password") && opts.Password == "" { print("Password: ") pass, err := gopass.GetPasswd() @@ -157,13 +187,14 @@ func main() { Source: opts.AuthDB, } + log.Debugf("Connecting to the db using:\n%+v", di) dialer := pmgo.NewDialer() hostnames, err := getHostnames(dialer, di) session, err := dialer.DialWithInfo(di) if err != nil { - log.Printf("cannot connect to the db: %s", err) + log.Errorf("cannot connect to the db: %s", err) os.Exit(1) } defer session.Close() @@ -199,7 +230,7 @@ func main() { t.Execute(os.Stdout, security) } - if oplogInfo, err := GetOplogInfo(hostnames, di); err != nil { + if oplogInfo, err := oplog.GetOplogInfo(hostnames, di); err != nil { log.Printf("[Error] cannot get Oplog info: %v\n", err) } else { if len(oplogInfo) > 0 { @@ -224,57 +255,6 @@ func main() { } -func GetHostinfo2(session pmgo.SessionManager) (*hostInfo, error) { - - hi := proto.HostInfo{} - if err := session.Run(bson.M{"hostInfo": 1}, &hi); err != nil { - return nil, errors.Wrap(err, "GetHostInfo.hostInfo") - } - - cmdOpts := proto.CommandLineOptions{} - err := session.DB("admin").Run(bson.D{{"getCmdLineOpts", 1}, {"recordStats", 1}}, &cmdOpts) - if err != nil { - return nil, errors.Wrap(err, "cannot get command line options") - } - - ss := proto.ServerStatus{} - if err := session.DB("admin").Run(bson.D{{"serverStatus", 1}, {"recordStats", 1}}, &ss); err != nil { - return nil, errors.Wrap(err, "GetHostInfo.serverStatus") - } - - pi := procInfo{} - if err := getProcInfo(int32(ss.Pid), &pi); err != nil { - pi.Error = err - } - - nodeType, _ := getNodeType(session) - - i := &hostInfo{ - Hostname: hi.System.Hostname, - HostOsType: hi.Os.Type, - HostSystemCPUArch: hi.System.CpuArch, - HostDatabases: hi.DatabasesCount, - HostCollections: hi.CollectionsCount, - DBPath: "", // Sets default. It will be overriden later if necessary - - ProcessName: ss.Process, - Version: ss.Version, - NodeType: nodeType, - - ProcPath: pi.Path, - ProcUserName: pi.UserName, - ProcCreateTime: pi.CreateTime, - } - if ss.Repl != nil { - i.ReplicasetName = ss.Repl.SetName - } - - if cmdOpts.Parsed.Storage.DbPath != "" { - i.DBPath = cmdOpts.Parsed.Storage.DbPath - } - - return i, nil -} func GetHostinfo(session pmgo.SessionManager) (*hostInfo, error) { hi := proto.HostInfo{} @@ -336,11 +316,14 @@ func getHostnames(dialer pmgo.Dialer, di *mgo.DialInfo) ([]string, error) { defer session.Close() shardsInfo := &proto.ShardsInfo{} + log.Debugf("Running 'listShards' command") err = session.Run("listShards", shardsInfo) if err != nil { return nil, errors.Wrap(err, "cannot list shards") } + log.Debugf("listShards raw response: %+v", util.Pretty(shardsInfo)) + hostnames := []string{di.Addrs[0]} if shardsInfo != nil { for _, shardInfo := range shardsInfo.Shards { diff --git a/src/go/pt-mongodb-summary/main_test.go b/src/go/pt-mongodb-summary/main_test.go index edf17f94..7eab103f 100644 --- a/src/go/pt-mongodb-summary/main_test.go +++ b/src/go/pt-mongodb-summary/main_test.go @@ -16,6 +16,7 @@ import ( ) func TestGetOpCounterStats(t *testing.T) { + ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -37,13 +38,16 @@ func TestGetOpCounterStats(t *testing.T) { session.EXPECT().DB("admin").Return(database) database.EXPECT().Run(bson.D{{"serverStatus", 1}, {"recordStats", 1}}, gomock.Any()).SetArg(1, ss) + session.EXPECT().DB("admin").Return(database) + database.EXPECT().Run(bson.D{{"serverStatus", 1}, {"recordStats", 1}}, gomock.Any()).SetArg(1, ss) + ss = addToCounters(ss, 1) session.EXPECT().DB("admin").Return(database) database.EXPECT().Run(bson.D{{"serverStatus", 1}, {"recordStats", 1}}, gomock.Any()).SetArg(1, ss) var sampleCount int64 = 5 var sampleRate time.Duration = 10 * time.Millisecond // in seconds - expect := timedStats{Min: 7, Max: 7, Total: 7, Avg: 1} + expect := TimedStats{Min: 0, Max: 0, Total: 0, Avg: 0} os, err := GetOpCountersStats(session, sampleCount, sampleRate) if err != nil { @@ -55,16 +59,6 @@ func TestGetOpCounterStats(t *testing.T) { } -func addToCounters(ss proto.ServerStatus, increment int64) proto.ServerStatus { - ss.Opcounters.Command += increment - ss.Opcounters.Delete += increment - ss.Opcounters.GetMore += increment - ss.Opcounters.Insert += increment - ss.Opcounters.Query += increment - ss.Opcounters.Update += increment - return ss -} - func TestSecurityOpts(t *testing.T) { cmdopts := []proto.CommandLineOptions{ // 1 @@ -381,3 +375,13 @@ func TestGetHostnames(t *testing.T) { t.Errorf("getHostnames: got %+v, expected: %+v\n", rss, expect) } } + +func addToCounters(ss proto.ServerStatus, increment int64) proto.ServerStatus { + ss.Opcounters.Command += increment + ss.Opcounters.Delete += increment + ss.Opcounters.GetMore += increment + ss.Opcounters.Insert += increment + ss.Opcounters.Query += increment + ss.Opcounters.Update += increment + return ss +} diff --git a/src/go/pt-mongodb-summary/oplog.go b/src/go/pt-mongodb-summary/oplog/oplog.go similarity index 99% rename from src/go/pt-mongodb-summary/oplog.go rename to src/go/pt-mongodb-summary/oplog/oplog.go index 62104e54..d5c3973e 100644 --- a/src/go/pt-mongodb-summary/oplog.go +++ b/src/go/pt-mongodb-summary/oplog/oplog.go @@ -1,4 +1,4 @@ -package main +package oplog import ( "fmt" diff --git a/src/go/pt-mongodb-summary/oplog/oplog_test.go b/src/go/pt-mongodb-summary/oplog/oplog_test.go new file mode 100644 index 00000000..b70c3351 --- /dev/null +++ b/src/go/pt-mongodb-summary/oplog/oplog_test.go @@ -0,0 +1 @@ +package oplog diff --git a/src/go/tests/lib/sample-config1.conf b/src/go/tests/lib/sample-config1.conf new file mode 100644 index 00000000..0ec00811 --- /dev/null +++ b/src/go/tests/lib/sample-config1.conf @@ -0,0 +1,9 @@ +no-version-check +trueboolvar=true +yesboolvar=yes +falseboolvar=false +noboolvar=no +intvar=1 +#ignored comment +floatvar=2.3 +stringvar=some string var having = and # diff --git a/src/go/tests/lib/sample-config2.conf b/src/go/tests/lib/sample-config2.conf new file mode 100644 index 00000000..93e5dbfb --- /dev/null +++ b/src/go/tests/lib/sample-config2.conf @@ -0,0 +1,10 @@ +trueboolvar=false +yesboolvar=no +falseboolvar=true +noboolvar=yes +intvar=4 +#ignored comment +floatvar=5.6 +stringvar=some other string +newstring=a new string + anotherint = 8