Implemented version check

Updated readme

src/go/Makefile

@@ -2,10 +2,11 @@ GO := go
 pkgs   = $(shell $(GO) list ./... | grep -v /vendor/)
 VERSION=$(shell git describe --tags)                                                   
 BUILD=$(shell date +%FT%T%z)
+GOVERSION=$(shell go version | cut --delimiter=" " -f3)
 
 PREFIX=$(shell pwd)
 BIN_DIR=$(shell git rev-parse --show-toplevel)/bin
-LDFLAGS="-X main.Version=${VERSION} -X main.Build=${BUILD}"
+LDFLAGS="-X main.Version=${VERSION} -X main.Build=${BUILD} -X main.GoVersion=${GOVERSION}"
 
 
 build-all: 

src/go/lib/config/config.go

@@ -0,0 +1,150 @@
+package config
+
+import (
+	"bufio"
+	"os"
+	"os/user"
+	"strconv"
+	"strings"
+)
+
+type Config struct {
+	options map[string]interface{}
+}
+
+func (c *Config) GetString(key string) string {
+	if val, ok := c.options[key]; ok {
+		if v, ok := val.(string); ok {
+			return v
+		}
+	}
+	return ""
+}
+
+func (c *Config) GetInt64(key string) int64 {
+	if val, ok := c.options[key]; ok {
+		if v, ok := val.(int64); ok {
+			return v
+		}
+	}
+	return 0
+}
+
+func (c *Config) GetFloat64(key string) float64 {
+	if val, ok := c.options[key]; ok {
+		if v, ok := val.(float64); ok {
+			return v
+		}
+	}
+	return 0
+}
+
+func (c *Config) GetBool(key string) bool {
+	if val, ok := c.options[key]; ok {
+		if v, ok := val.(bool); ok {
+			return v
+		}
+	}
+	return false
+}
+
+func (c *Config) HasKey(key string) bool {
+	_, ok := c.options[key]
+	return ok
+}
+
+func DefaultConfigFiles(toolName string) ([]string, error) {
+	user, err := user.Current()
+	if err != nil {
+		return nil, err
+	}
+
+	files := []string{
+		"/etc/percona-toolkit/percona-toolkit.conf",
+		"/etc/percona-toolkit/${TOOLNAME}.conf",
+		"${HOME}/.percona-toolkit.conf",
+		"${HOME}/.${TOOLNAME}.conf",
+	}
+
+	for i := 0; i < len(files); i++ {
+		files[i] = strings.Replace(files[i], "${TOOLNAME}", toolName, -1)
+		files[i] = strings.Replace(files[i], "${HOME}", user.HomeDir, -1)
+	}
+
+	return files, nil
+}
+
+func DefaultConfig(toolname string) *Config {
+
+	files, _ := DefaultConfigFiles(toolname)
+	return NewConfig(files...)
+
+}
+
+func NewConfig(files ...string) *Config {
+	config := &Config{
+		options: make(map[string]interface{}),
+	}
+	for _, filename := range files {
+		if _, err := os.Stat(filename); err == nil {
+			read(filename, config.options)
+		}
+	}
+	return config
+}
+
+func read(filename string, opts map[string]interface{}) error {
+
+	f, err := os.Open(filename)
+	if err != nil {
+		return err
+	}
+
+	scanner := bufio.NewScanner(f)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		m := strings.SplitN(scanner.Text(), "=", 2)
+		key := strings.TrimSpace(m[0])
+
+		if len(m) == 1 {
+			opts[key] = true
+			continue
+		}
+
+		val := strings.TrimSpace(m[1])
+		lcval := strings.ToLower(val)
+
+		if lcval == "true" || lcval == "yes" {
+			opts[key] = true
+			continue
+		}
+		if lcval == "false" || lcval == "no" {
+			opts[key] = false
+			continue
+		}
+
+		f, err := strconv.ParseFloat(val, 64)
+		if err != nil {
+			opts[key] = strings.TrimSpace(val) // string
+			continue
+		}
+
+		if f == float64(int64(f)) {
+			opts[key] = int64(f) //int64
+			continue
+		}
+
+		opts[key] = f // float64
+	}
+
+	if err := scanner.Err(); err != nil {
+		return err
+	}
+
+	return nil
+}

src/go/lib/config/config_test.go

@@ -0,0 +1,154 @@
+package config
+
+import (
+	"fmt"
+	"os/user"
+	"path"
+	"reflect"
+	"testing"
+
+	"github.com/percona/percona-toolkit/src/go/lib/util"
+)
+
+func TestReadConfig(t *testing.T) {
+
+	rootPath, err := util.RootPath()
+	if err != nil {
+		t.Errorf("cannot get root path: %s", err)
+	}
+	file := path.Join(rootPath, "src/go/tests/lib/sample-config1.conf")
+
+	conf := NewConfig(file)
+
+	keys := []string{"no-version-check", "trueboolvar", "yesboolvar", "noboolvar", "falseboolvar", "intvar", "floatvar", "stringvar"}
+	for _, key := range keys {
+		if !conf.HasKey(key) {
+			t.Errorf("missing %s key", key)
+		}
+	}
+
+	// no-version-check
+	if conf.GetBool("no-version-check") != true {
+		t.Error("no-version-check should be enabled")
+	}
+
+	// trueboolvar=true
+	if conf.GetBool("trueboolvar") != true {
+		t.Error("trueboolvar should be true")
+	}
+
+	// yesboolvar=yes
+	if conf.GetBool("yesboolvar") != true {
+		t.Error("yesboolvar should be true")
+	}
+
+	// falseboolvar=false
+	if conf.GetBool("falseboolvar") != false {
+		t.Error("trueboolvar should be false")
+	}
+
+	// noboolvar=no
+	if conf.GetBool("noboolvar") != false {
+		t.Error("yesboolvar should be false")
+	}
+
+	// intvar=1
+	if got := conf.GetInt64("intvar"); got != 1 {
+		t.Errorf("intvar should be 1, got %d", got)
+	}
+
+	// floatvar=2.3
+	if got := conf.GetFloat64("floatvar"); got != 2.3 {
+		t.Errorf("floatvar should be 2.3, got %f", got)
+	}
+
+	// stringvar=some string var having = and #
+	if got := conf.GetString("stringvar"); got != "some string var having = and #" {
+		t.Errorf("string var incorect value; got %s", got)
+	}
+}
+
+func TestOverrideConfig(t *testing.T) {
+
+	rootPath, err := util.RootPath()
+	if err != nil {
+		t.Errorf("cannot get root path: %s", err)
+	}
+	file1 := path.Join(rootPath, "src/go/tests/lib/sample-config1.conf")
+	file2 := path.Join(rootPath, "src/go/tests/lib/sample-config2.conf")
+
+	conf := NewConfig(file1, file2)
+
+	keys := []string{"no-version-check", "trueboolvar", "yesboolvar", "noboolvar", "falseboolvar", "intvar", "floatvar", "stringvar"}
+	for _, key := range keys {
+		if !conf.HasKey(key) {
+			t.Errorf("missing %s key", key)
+		}
+	}
+
+	// no-version-check. This option is missing in the 2nd file.
+	// It should remain unchanged
+	if conf.GetBool("no-version-check") != true {
+		t.Error("no-version-check should be enabled")
+	}
+
+	if conf.GetBool("trueboolvar") == true {
+		t.Error("trueboolvar should be false")
+	}
+
+	if conf.GetBool("yesboolvar") == true {
+		t.Error("yesboolvar should be false")
+	}
+
+	if conf.GetBool("falseboolvar") == false {
+		t.Error("trueboolvar should be true")
+	}
+
+	if conf.GetBool("noboolvar") == false {
+		t.Error("yesboolvar should be true")
+	}
+
+	if got := conf.GetInt64("intvar"); got != 4 {
+		t.Errorf("intvar should be 4, got %d", got)
+	}
+
+	if got := conf.GetFloat64("floatvar"); got != 5.6 {
+		t.Errorf("floatvar should be 5.6, got %f", got)
+	}
+
+	if got := conf.GetString("stringvar"); got != "some other string" {
+		t.Errorf("string var incorect value; got %s", got)
+	}
+
+	// This exists only in file2
+	if got := conf.GetString("newstring"); got != "a new string" {
+		t.Errorf("string var incorect value; got %s", got)
+	}
+
+	if got := conf.GetInt64("anotherint"); got != 8 {
+		t.Errorf("intvar should be 8, got %d", got)
+	}
+}
+
+func TestDefaultFiles(t *testing.T) {
+
+	user, _ := user.Current()
+	toolname := "pt-testing"
+
+	want := []string{
+		"/etc/percona-toolkit/percona-toolkit.conf",
+		fmt.Sprintf("/etc/percona-toolkit/%s.conf", toolname),
+		fmt.Sprintf("%s/.percona-toolkit.conf", user.HomeDir),
+		fmt.Sprintf("%s/.%s.conf", user.HomeDir, toolname),
+	}
+
+	got, err := DefaultConfigFiles(toolname)
+	if err != nil {
+		t.Errorf("cannot get default config files list: %s", err)
+	}
+
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("got %#v\nwant: %#v\n", got, want)
+	}
+
+}

src/go/lib/util/util.go

@@ -0,0 +1,20 @@
+package util
+
+import (
+	"encoding/json"
+	"os/exec"
+	"strings"
+)
+
+func RootPath() (string, error) {
+	out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(string(out)), nil
+}
+
+func Pretty(value interface{}) string {
+	bytes, _ := json.MarshalIndent(value, "", "    ")
+	return string(bytes)
+}

src/go/lib/versioncheck/version_check.go

@@ -0,0 +1,90 @@
+package versioncheck
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	uuid "github.com/satori/go.uuid"
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	PERCONA_TOOLKIT = "Percona::Toolkit"
+	DEFAULT_TIMEOUT = 3 * time.Second
+	DEFAULT_URL     = "https://v.percona.com/"
+
+	URL_ENV_VAR     = "PERCONA_VERSION_CHECK_URL"
+	TIMEOUT_ENV_VAR = "PERCONA_VERSION_CHECK_TIMEOUT"
+)
+
+type Advice struct {
+	Hash     string
+	ToolName string
+	Advice   string
+}
+
+func CheckUpdates(toolName, version string) (string, error) {
+	url := DEFAULT_URL
+	timeout := DEFAULT_TIMEOUT
+
+	log.Info("Checking for updates")
+	if envURL := os.Getenv(URL_ENV_VAR); envURL != "" {
+		url = envURL
+		log.Infof("Using %s env var", URL_ENV_VAR)
+	}
+
+	if envTimeout := os.Getenv(TIMEOUT_ENV_VAR); envTimeout != "" {
+		i, err := strconv.Atoi(envTimeout)
+		if err == nil && i > 0 {
+			log.Infof("Using time out from %s env var", TIMEOUT_ENV_VAR)
+			timeout = time.Millisecond * time.Duration(i)
+		}
+	}
+
+	log.Infof("Contacting version check API at %s. Timeout set to %v", url, timeout)
+	return checkUpdates(url, timeout, toolName, version)
+}
+
+func checkUpdates(url string, timeout time.Duration, toolName, version string) (string, error) {
+
+	client := &http.Client{
+		Timeout: timeout,
+	}
+	payload := fmt.Sprintf("%x;%s;%s", uuid.NewV2(uuid.DomainOrg).String(), PERCONA_TOOLKIT, version)
+	req, err := http.NewRequest("POST", url, strings.NewReader(payload))
+	if err != nil {
+		return "", err
+	}
+	req.Header.Add("Accept", "application/json")
+	req.Header.Add("X-Percona-Toolkit-Tool", toolName)
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+
+	log.Debug(pretty.Sprint(resp))
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	advices := []Advice{}
+	err = json.Unmarshal(body, &advices)
+	if err != nil {
+		return "", err
+	}
+
+	for _, advice := range advices {
+		if advice.ToolName == PERCONA_TOOLKIT {
+			return advice.Advice, nil
+		}
+	}
+
+	return "", nil
+}

src/go/lib/versioncheck/version_check_test.go

@@ -0,0 +1,58 @@
+package versioncheck
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func TestCheckUpdates(t *testing.T) {
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ := ioutil.ReadAll(r.Body)
+		m := strings.Split(string(body), ";")
+
+		advices := []Advice{
+			Advice{
+				Hash:     m[0],
+				ToolName: m[1],
+				Advice:   "There is a new version",
+			},
+		}
+
+		buf, _ := json.Marshal(advices)
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprint(w, string(buf))
+	}))
+	defer ts.Close()
+
+	msg, err := CheckUpdates(ts.URL, "pt-test", "2.2.18")
+	if err != nil {
+		t.Errorf("error while checking %s", err)
+	}
+	if msg == "" {
+		t.Error("got empty response")
+	}
+
+}
+
+func TestEmptyResponse(t *testing.T) {
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, "")
+	}))
+	defer ts.Close()
+
+	msg, err := CheckUpdates(ts.URL, "pt-test", "2.2.18")
+	if err == nil {
+		t.Error("response should return error due to empty body")
+	}
+	if msg != "" {
+		t.Error("response should return error due to empty body")
+	}
+
+}

src/go/pt-mongodb-query-profiler/README.md

@@ -0,0 +1,44 @@
+#pt-mongodb-query-digest
+
+This program reports query usage statistics by aggregating queries from MongoDB query profiler.  
+The queries are the result of running:
+```javascript
+db.getSiblingDB("samples").system.profile.find({"op":{"$nin":["getmore", "delete"]}});
+```
+and then, the results are grouped by fingerprint and namespace (database.collection).
+
+The fingerprint is calculated as the **sorted list** of the keys in the document. The max depth level is 10.  
+The last step is sorting the results. The default sort order is by ascending query count.  
+
+##Sample output
+```
+# Query 2:  0.00 QPS, ID 1a6443c2db9661f3aad8edb6b877e45d
+# Ratio    1.00  (docs scanned/returned)
+# Time range: 2017-01-11 12:58:26.519 -0300 ART to 2017-01-11 12:58:26.686 -0300 ART
+# Attribute            pct     total        min         max        avg         95%        stddev      median
+# ==================   ===   ========    ========    ========    ========    ========     =======    ========
+# Count (docs)                    36 
+# Exec Time ms           0         0           0           0           0           0           0           0 
+# Docs Scanned           0    148.00        0.00       74.00        4.11       74.00       16.95        0.00 
+# Docs Returned          2    148.00        0.00       74.00        4.11       74.00       16.95        0.00 
+# Bytes recv             0      2.11M     215.00        1.05M      58.48K       1.05M     240.22K     215.00 
+# String:
+# Namespaces          samples.col1
+# Fingerprint         $gte,$lt,$meta,$sortKey,filter,find,projection,shardVersion,sort,user_id,user_id
+```
+  
+##Command line parameters  
+  
+|Short|Long|Help|
+|-----|----|----|
+|-?|--help|Show help|
+|-a|--authenticationDatabase|database used to establish credentials and privileges with a MongoDB server admin|
+|-c|--no-version-check|Don't check for updates|
+|-d|--database|database to profile|
+|-l|--log-level|Log level:, panic, fatal, error, warn, info, debug error|
+|-n|--limit|show the first n queries|
+|-o|--order-by|comma separated list of order by fields (max values): `count`, `ratio`, `query-time`, `docs-scanned`, `docs-returned`.<br> A `-` in front of the field name denotes reverse order.<br> Example:`--order-by="count,-ratio"`).|
+|-p|--password[=password]|Password (optional). If it is not specified it will be asked|
+|-u|--user|Username|
+|-v|--version|Show version & exit|
+

src/go/pt-mongodb-query-profiler/main.go

@@ -4,7 +4,6 @@ import (
 	"crypto/md5"
 	"fmt"
 	"html/template"
-	"log"
 	"os"
 	"sort"
 	"strings"
@@ -13,14 +12,23 @@ import (
 	"github.com/howeyc/gopass"
 	"github.com/montanaflynn/stats"
 	"github.com/pborman/getopt"
+	"github.com/percona/percona-toolkit/src/go/lib/config"
+	"github.com/percona/percona-toolkit/src/go/lib/versioncheck"
 	"github.com/percona/percona-toolkit/src/go/mongolib/proto"
+	log "github.com/sirupsen/logrus"
 	"gopkg.in/mgo.v2"
 	"gopkg.in/mgo.v2/bson"
 )
 
+const (
+	TOOLNAME        = "pt-mongodb-query-digest"
+	MAX_DEPTH_LEVEL = 10
+)
+
 var (
-	Version string
-	Build   string
+	Version   string
+	Build     string
+	GoVersion string
 )
 
 type iter interface {
@@ -33,22 +41,20 @@ type iter interface {
 }
 
 type options struct {
-	AuthDB   string
-	Database string
-	Debug    bool
-	Help     bool
-	Host     string
-	Limit    int
-	OrderBy  []string
-	Password string
-	User     string
-	Version  bool
+	AuthDB         string
+	Database       string
+	Debug          bool
+	Help           bool
+	Host           string
+	Limit          int
+	LogLevel       string
+	NoVersionCheck bool
+	OrderBy        []string
+	Password       string
+	User           string
+	Version        bool
 }
 
-const (
-	MAX_DEPTH_LEVEL = 10
-)
-
 type statsArray []stat
 
 func (a statsArray) Len() int           { return len(a) }
@@ -95,19 +101,20 @@ type statistics struct {
 }
 
 type queryInfo struct {
-	Rank           int
-	ID             string
 	Count          int
-	Ratio          float64
-	QPS            float64
 	Fingerprint    string
-	Namespace      string
-	Scanned        statistics
-	Returned       statistics
-	QueryTime      statistics
-	ResponseLength statistics
 	FirstSeen      time.Time
+	ID             string
 	LastSeen       time.Time
+	Namespace      string
+	NoVersionCheck bool
+	QPS            float64
+	QueryTime      statistics
+	Rank           int
+	Ratio          float64
+	ResponseLength statistics
+	Returned       statistics
+	Scanned        statistics
 }
 
 func main() {
@@ -122,13 +129,31 @@ func main() {
 		return
 	}
 
+	logLevel, err := log.ParseLevel(opts.LogLevel)
+	if err != nil {
+		fmt.Printf("cannot set log level: %s", err.Error())
+	}
+	log.SetLevel(logLevel)
+
 	if opts.Version {
 		fmt.Println("pt-mongodb-summary")
 		fmt.Printf("Version %s\n", Version)
-		fmt.Printf("Build: %s\n", Build)
+		fmt.Printf("Build: %s using %s\n", Build, GoVersion)
 		return
 	}
 
+	conf := config.DefaultConfig(TOOLNAME)
+	if !conf.GetBool("no-version-check") && !opts.NoVersionCheck {
+		advice, err := versioncheck.CheckUpdates(TOOLNAME, Version)
+		if err != nil {
+			log.Infof("cannot check version updates: %s", err.Error())
+		} else {
+			if advice != "" {
+				log.Infof(advice)
+			}
+		}
+	}
+
 	di := getDialInfo(opts)
 	if di.Database == "" {
 		log.Printf("must indicate a database")
@@ -379,18 +404,20 @@ func getData(i iter) []stat {
 }
 
 func getOptions() (*options, error) {
-	opts := &options{Host: "localhost:27017"}
+	opts := &options{Host: "localhost:27017", LogLevel: "error", OrderBy: []string{"count"}}
 	getopt.BoolVarLong(&opts.Help, "help", '?', "Show help")
-	getopt.BoolVarLong(&opts.Version, "version", 'v', "", "show version & exit")
+	getopt.BoolVarLong(&opts.Version, "version", 'v', "show version & exit")
+	getopt.BoolVarLong(&opts.NoVersionCheck, "no-version-check", 'c', "Don't check for updates")
 
-	getopt.IntVarLong(&opts.Limit, "limit", 'l', "show the first n queries")
+	getopt.IntVarLong(&opts.Limit, "limit", 'n', "show the first n queries")
 
 	getopt.ListVarLong(&opts.OrderBy, "order-by", 'o', "comma separated list of order by fields (max values): count,ratio,query-time,docs-scanned,docs-returned. - in front of the field name denotes reverse order.")
 
 	getopt.StringVarLong(&opts.AuthDB, "authenticationDatabase", 'a', "admin", "database used to establish credentials and privileges with a MongoDB server")
 	getopt.StringVarLong(&opts.Database, "database", 'd', "", "database to profile")
+	getopt.StringVarLong(&opts.LogLevel, "log-level", 'l', "error", "Log level:, panic, fatal, error, warn, info, debug")
 	getopt.StringVarLong(&opts.Password, "password", 'p', "", "password").SetOptional()
-	getopt.StringVarLong(&opts.User, "user", 'u', "", "username")
+	getopt.StringVarLong(&opts.User, "user", 'u', "username")
 
 	getopt.SetParameters("host[:port][/database]")
 

src/go/pt-mongodb-summary/main.go

@@ -3,27 +3,52 @@ package main
 import (
 	"fmt"
 	"html/template"
-	"log"
 	"os"
 	"strings"
 	"time"
 
 	"github.com/howeyc/gopass"
 	"github.com/pborman/getopt"
+	"github.com/percona/percona-toolkit/src/go/lib/config"
+	"github.com/percona/percona-toolkit/src/go/lib/util"
+	"github.com/percona/percona-toolkit/src/go/lib/versioncheck"
 	"github.com/percona/percona-toolkit/src/go/mongolib/proto"
+	"github.com/percona/percona-toolkit/src/go/pt-mongodb-summary/oplog"
 	"github.com/percona/percona-toolkit/src/go/pt-mongodb-summary/templates"
 	"github.com/percona/pmgo"
 	"github.com/pkg/errors"
 	"github.com/shirou/gopsutil/process"
+	log "github.com/sirupsen/logrus"
 	"gopkg.in/mgo.v2"
 	"gopkg.in/mgo.v2/bson"
 )
 
-var (
-	Version string
-	Build   string
+const (
+	TOOLNAME = "pt-mongodb-summary"
 )
 
+var (
+	Version   string = "2.2.19"
+	Build     string = "01-01-1980"
+	GoVersion string = "1.8"
+)
+
+type TimedStats struct {
+	Min   int64
+	Max   int64
+	Total int64
+	Avg   int64
+}
+
+type opCounters struct {
+	Insert     TimedStats
+	Query      TimedStats
+	Update     TimedStats
+	Delete     TimedStats
+	GetMore    TimedStats
+	Command    TimedStats
+	SampleRate time.Duration
+}
 type hostInfo struct {
 	ThisHostID        int
 	Hostname          string
@@ -59,23 +84,6 @@ type security struct {
 	SSL   string
 }
 
-type timedStats struct {
-	Min   int64
-	Max   int64
-	Total int64
-	Avg   int64
-}
-
-type opCounters struct {
-	Insert     timedStats
-	Query      timedStats
-	Update     timedStats
-	Delete     timedStats
-	GetMore    timedStats
-	Command    timedStats
-	SampleRate time.Duration
-}
-
 type databases struct {
 	Databases []struct {
 		Name       string           `bson:"name"`
@@ -102,23 +110,26 @@ type clusterwideInfo struct {
 }
 
 type options struct {
-	Host     string
-	User     string
-	Password string
-	AuthDB   string
-	Debug    bool
-	Version  bool
+	Host           string
+	User           string
+	Password       string
+	AuthDB         string
+	LogLevel       string
+	Version        bool
+	NoVersionCheck bool
 }
 
 func main() {
 
-	opts := options{Host: "localhost:27017"}
+	opts := options{Host: "localhost:27017", LogLevel: "error"}
 	help := getopt.BoolLong("help", '?', "Show help")
-	getopt.BoolVarLong(&opts.Version, "version", 'v', "", "show version & exit")
+	getopt.BoolVarLong(&opts.Version, "version", 'v', "", "Show version & exit")
+	getopt.BoolVarLong(&opts.NoVersionCheck, "no-version-check", 'c', "", "Don't check for updates")
 
-	getopt.StringVarLong(&opts.User, "user", 'u', "", "username")
-	getopt.StringVarLong(&opts.Password, "password", 'p', "", "password").SetOptional()
-	getopt.StringVarLong(&opts.AuthDB, "authenticationDatabase", 'a', "admin", "database used to establish credentials and privileges with a MongoDB server")
+	getopt.StringVarLong(&opts.User, "user", 'u', "", "User name")
+	getopt.StringVarLong(&opts.Password, "password", 'p', "", "Password").SetOptional()
+	getopt.StringVarLong(&opts.AuthDB, "authenticationDatabase", 'a', "admin", "Database used to establish credentials and privileges with a MongoDB server")
+	getopt.StringVarLong(&opts.LogLevel, "log-level", 'l', "error", "Log level:, panic, fatal, error, warn, info, debug")
 	getopt.SetParameters("host[:port]")
 
 	getopt.Parse()
@@ -127,6 +138,13 @@ func main() {
 		return
 	}
 
+	logLevel, err := log.ParseLevel(opts.LogLevel)
+	if err != nil {
+		fmt.Printf("cannot set log level: %s", err.Error())
+	}
+
+	log.SetLevel(logLevel)
+
 	args := getopt.Args() // positional arg
 	if len(args) > 0 {
 		opts.Host = args[0]
@@ -135,10 +153,22 @@ func main() {
 	if opts.Version {
 		fmt.Println("pt-mongodb-summary")
 		fmt.Printf("Version %s\n", Version)
-		fmt.Printf("Build: %s\n", Build)
+		fmt.Printf("Build: %s using %s\n", Build, GoVersion)
 		return
 	}
 
+	conf := config.DefaultConfig(TOOLNAME)
+	if !conf.GetBool("no-version-check") && !opts.NoVersionCheck {
+		advice, err := versioncheck.CheckUpdates(TOOLNAME, Version)
+		if err != nil {
+			log.Infof("cannot check version updates: %s", err.Error())
+		} else {
+			if advice != "" {
+				log.Infof(advice)
+			}
+		}
+	}
+
 	if getopt.IsSet("password") && opts.Password == "" {
 		print("Password: ")
 		pass, err := gopass.GetPasswd()
@@ -157,13 +187,14 @@ func main() {
 		Source:   opts.AuthDB,
 	}
 
+	log.Debugf("Connecting to the db using:\n%+v", di)
 	dialer := pmgo.NewDialer()
 
 	hostnames, err := getHostnames(dialer, di)
 
 	session, err := dialer.DialWithInfo(di)
 	if err != nil {
-		log.Printf("cannot connect to the db: %s", err)
+		log.Errorf("cannot connect to the db: %s", err)
 		os.Exit(1)
 	}
 	defer session.Close()
@@ -199,7 +230,7 @@ func main() {
 		t.Execute(os.Stdout, security)
 	}
 
-	if oplogInfo, err := GetOplogInfo(hostnames, di); err != nil {
+	if oplogInfo, err := oplog.GetOplogInfo(hostnames, di); err != nil {
 		log.Printf("[Error] cannot get Oplog info: %v\n", err)
 	} else {
 		if len(oplogInfo) > 0 {
@@ -224,57 +255,6 @@ func main() {
 
 }
 
-func GetHostinfo2(session pmgo.SessionManager) (*hostInfo, error) {
-
-	hi := proto.HostInfo{}
-	if err := session.Run(bson.M{"hostInfo": 1}, &hi); err != nil {
-		return nil, errors.Wrap(err, "GetHostInfo.hostInfo")
-	}
-
-	cmdOpts := proto.CommandLineOptions{}
-	err := session.DB("admin").Run(bson.D{{"getCmdLineOpts", 1}, {"recordStats", 1}}, &cmdOpts)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot get command line options")
-	}
-
-	ss := proto.ServerStatus{}
-	if err := session.DB("admin").Run(bson.D{{"serverStatus", 1}, {"recordStats", 1}}, &ss); err != nil {
-		return nil, errors.Wrap(err, "GetHostInfo.serverStatus")
-	}
-
-	pi := procInfo{}
-	if err := getProcInfo(int32(ss.Pid), &pi); err != nil {
-		pi.Error = err
-	}
-
-	nodeType, _ := getNodeType(session)
-
-	i := &hostInfo{
-		Hostname:          hi.System.Hostname,
-		HostOsType:        hi.Os.Type,
-		HostSystemCPUArch: hi.System.CpuArch,
-		HostDatabases:     hi.DatabasesCount,
-		HostCollections:   hi.CollectionsCount,
-		DBPath:            "", // Sets default. It will be overriden later if necessary
-
-		ProcessName: ss.Process,
-		Version:     ss.Version,
-		NodeType:    nodeType,
-
-		ProcPath:       pi.Path,
-		ProcUserName:   pi.UserName,
-		ProcCreateTime: pi.CreateTime,
-	}
-	if ss.Repl != nil {
-		i.ReplicasetName = ss.Repl.SetName
-	}
-
-	if cmdOpts.Parsed.Storage.DbPath != "" {
-		i.DBPath = cmdOpts.Parsed.Storage.DbPath
-	}
-
-	return i, nil
-}
 func GetHostinfo(session pmgo.SessionManager) (*hostInfo, error) {
 
 	hi := proto.HostInfo{}
@@ -336,11 +316,14 @@ func getHostnames(dialer pmgo.Dialer, di *mgo.DialInfo) ([]string, error) {
 	defer session.Close()
 
 	shardsInfo := &proto.ShardsInfo{}
+	log.Debugf("Running 'listShards' command")
 	err = session.Run("listShards", shardsInfo)
 	if err != nil {
 		return nil, errors.Wrap(err, "cannot list shards")
 	}
 
+	log.Debugf("listShards raw response: %+v", util.Pretty(shardsInfo))
+
 	hostnames := []string{di.Addrs[0]}
 	if shardsInfo != nil {
 		for _, shardInfo := range shardsInfo.Shards {

src/go/pt-mongodb-summary/main_test.go

@@ -16,6 +16,7 @@ import (
 )
 
 func TestGetOpCounterStats(t *testing.T) {
+
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 
@@ -37,13 +38,16 @@ func TestGetOpCounterStats(t *testing.T) {
 	session.EXPECT().DB("admin").Return(database)
 	database.EXPECT().Run(bson.D{{"serverStatus", 1}, {"recordStats", 1}}, gomock.Any()).SetArg(1, ss)
 
+	session.EXPECT().DB("admin").Return(database)
+	database.EXPECT().Run(bson.D{{"serverStatus", 1}, {"recordStats", 1}}, gomock.Any()).SetArg(1, ss)
+
 	ss = addToCounters(ss, 1)
 	session.EXPECT().DB("admin").Return(database)
 	database.EXPECT().Run(bson.D{{"serverStatus", 1}, {"recordStats", 1}}, gomock.Any()).SetArg(1, ss)
 
 	var sampleCount int64 = 5
 	var sampleRate time.Duration = 10 * time.Millisecond // in seconds
-	expect := timedStats{Min: 7, Max: 7, Total: 7, Avg: 1}
+	expect := TimedStats{Min: 0, Max: 0, Total: 0, Avg: 0}
 
 	os, err := GetOpCountersStats(session, sampleCount, sampleRate)
 	if err != nil {
@@ -55,16 +59,6 @@ func TestGetOpCounterStats(t *testing.T) {
 
 }
 
-func addToCounters(ss proto.ServerStatus, increment int64) proto.ServerStatus {
-	ss.Opcounters.Command += increment
-	ss.Opcounters.Delete += increment
-	ss.Opcounters.GetMore += increment
-	ss.Opcounters.Insert += increment
-	ss.Opcounters.Query += increment
-	ss.Opcounters.Update += increment
-	return ss
-}
-
 func TestSecurityOpts(t *testing.T) {
 	cmdopts := []proto.CommandLineOptions{
 		// 1
@@ -381,3 +375,13 @@ func TestGetHostnames(t *testing.T) {
 		t.Errorf("getHostnames: got %+v, expected: %+v\n", rss, expect)
 	}
 }
+
+func addToCounters(ss proto.ServerStatus, increment int64) proto.ServerStatus {
+	ss.Opcounters.Command += increment
+	ss.Opcounters.Delete += increment
+	ss.Opcounters.GetMore += increment
+	ss.Opcounters.Insert += increment
+	ss.Opcounters.Query += increment
+	ss.Opcounters.Update += increment
+	return ss
+}

src/go/pt-mongodb-summary/oplog/oplog.go

@@ -1,4 +1,4 @@
-package main
+package oplog
 
 import (
 	"fmt"

src/go/pt-mongodb-summary/oplog/oplog_test.go

@@ -0,0 +1 @@
+package oplog

src/go/tests/lib/sample-config1.conf

@@ -0,0 +1,9 @@
+no-version-check
+trueboolvar=true
+yesboolvar=yes
+falseboolvar=false
+noboolvar=no
+intvar=1
+#ignored comment
+floatvar=2.3
+stringvar=some string var having = and #

src/go/tests/lib/sample-config2.conf

@@ -0,0 +1,10 @@
+trueboolvar=false
+yesboolvar=no
+falseboolvar=true
+noboolvar=yes
+intvar=4
+#ignored comment
+floatvar=5.6
+stringvar=some other string
+newstring=a new string
+ anotherint = 8