Fixes for PT-70, PT-68 & PT-69

src/go/pt-mongodb-query-digest/main.go

@@ -189,9 +189,15 @@ func main() {
 	}
 
 	if isProfilerEnabled == false {
+		count, err := systemProfileDocsCount(session, di.Database)
+		if err != nil || count == 0 {
 			log.Error("Profiler is not enabled")
 			os.Exit(5)
 		}
+		fmt.Printf("Profiler is disabled for the %q database but there are %d documents in the system.profile collection.\n",
+			di.Database, count)
+		fmt.Println("Using those documents for the stats")
+	}
 
 	filters := []docsFilter{}
 
@@ -487,19 +493,22 @@ func getOptions() (*options, error) {
 	}
 
 	getopt.BoolVarLong(&opts.Help, "help", '?', "Show help")
-	getopt.BoolVarLong(&opts.Version, "version", 'v', "show version & exit")
+	getopt.BoolVarLong(&opts.Version, "version", 'v', "Show version & exit")
-	getopt.BoolVarLong(&opts.NoVersionCheck, "no-version-check", 'c', "Don't check for updates")
+	getopt.BoolVarLong(&opts.NoVersionCheck, "no-version-check", 'c', "Default: Don't check for updates")
 
-	getopt.IntVarLong(&opts.Limit, "limit", 'n', "show the first n queries")
+	getopt.IntVarLong(&opts.Limit, "limit", 'n', "Show the first n queries")
 
-	getopt.ListVarLong(&opts.OrderBy, "order-by", 'o', "comma separated list of order by fields (max values): count,ratio,query-time,docs-scanned,docs-returned. - in front of the field name denotes reverse order.")
+	getopt.ListVarLong(&opts.OrderBy, "order-by", 'o', "Comma separated list of order by fields (max values): "+
-	getopt.ListVarLong(&opts.SkipCollections, "skip-collections", 's', "comma separated list of collections (namespaces) to skip. Default: system.profile")
+		"count,ratio,query-time,docs-scanned,docs-returned. "+
+		"- in front of the field name denotes reverse order.")
+	getopt.ListVarLong(&opts.SkipCollections, "skip-collections", 's', "A comma separated list of collections (namespaces) to skip."+
+		"  Default: system.profile")
 
-	getopt.StringVarLong(&opts.AuthDB, "authenticationDatabase", 'a', "admin", "database used to establish credentials and privileges with a MongoDB server")
+	getopt.StringVarLong(&opts.AuthDB, "authenticationDatabase", 'a', "admin", "Databaase to use for optional MongoDB authentication. Default: admin")
-	getopt.StringVarLong(&opts.Database, "database", 'd', "", "database to profile")
+	getopt.StringVarLong(&opts.Database, "database", 'd', "", "MongoDB database to profile")
-	getopt.StringVarLong(&opts.LogLevel, "log-level", 'l', "error", "Log level:, panic, fatal, error, warn, info, debug")
+	getopt.StringVarLong(&opts.LogLevel, "log-level", 'l', "Log level: error", "panic, fatal, error, warn, info, debug. Default: error")
-	getopt.StringVarLong(&opts.Password, "password", 'p', "", "password").SetOptional()
+	getopt.StringVarLong(&opts.Password, "password", 'p', "", "Password to use for optional MongoDB authentication").SetOptional()
-	getopt.StringVarLong(&opts.User, "user", 'u', "username")
+	getopt.StringVarLong(&opts.User, "username", 'u', "Username to use for optional MongoDB authentication")
 
 	getopt.SetParameters("host[:port]/database")
 
@@ -547,7 +556,7 @@ func getDialInfo(opts *options) *mgo.DialInfo {
 	di, _ := mgo.ParseURL(opts.Host)
 	di.FailFast = true
 
-	if getopt.IsSet("user") {
+	if getopt.IsSet("username") {
 		di.Username = opts.User
 	}
 	if getopt.IsSet("password") {
@@ -571,6 +580,9 @@ func fingerprint(query map[string]interface{}) string {
 func keys(query map[string]interface{}, level int) []string {
 	ks := []string{}
 	for key, value := range query {
+		if !shouldIncludeKey(key) {
+			continue
+		}
 		ks = append(ks, key)
 		if m, ok := value.(map[string]interface{}); ok {
 			level++
@@ -583,10 +595,20 @@ func keys(query map[string]interface{}, level int) []string {
 	return ks
 }
 
+func shouldIncludeKey(key string) bool {
+	filterOut := []string{"shardVersion"}
+	for _, val := range filterOut {
+		if val == key {
+			return false
+		}
+	}
+	return true
+}
+
 func printHeader(opts *options) {
 	fmt.Printf("%s - %s\n", TOOLNAME, time.Now().Format(time.RFC1123Z))
 	fmt.Printf("Host: %s\n", opts.Host)
-	fmt.Printf("Skipping docs in these collections: %v\n", opts.SkipCollections)
+	fmt.Printf("Skipping profiled queries on these collections: %v\n", opts.SkipCollections)
 	fmt.Println("")
 }
 
@@ -778,15 +800,26 @@ func isProfilerEnabled(dialer pmgo.Dialer, di *mgo.DialInfo) (bool, error) {
 	if err != nil {
 		return false, err
 	}
+
 	for _, member := range replicaMembers {
-		if member.State == proto.REPLICA_SET_MEMBER_PRIMARY {
+		// Stand alone instances return state = REPLICA_SET_MEMBER_STARTUP
 		di.Addrs = []string{member.Name}
 		session, err := dialer.DialWithInfo(di)
 		if err != nil {
 			continue
 		}
 		defer session.Close()
+		session.SetMode(mgo.Monotonic, true)
 
+		isReplicaEnabled := isReplicasetEnabled(session)
+
+		if member.StateStr == "configsvr" {
+			continue
+		}
+
+		if isReplicaEnabled && member.State != proto.REPLICA_SET_MEMBER_PRIMARY {
+			continue
+		}
 		if err := session.DB(di.Database).Run(bson.M{"profile": -1}, &ps); err != nil {
 			continue
 		}
@@ -794,6 +827,17 @@ func isProfilerEnabled(dialer pmgo.Dialer, di *mgo.DialInfo) (bool, error) {
 			return false, nil
 		}
 	}
-	}
 	return true, nil
 }
+
+func systemProfileDocsCount(session pmgo.SessionManager, dbname string) (int, error) {
+	return session.DB(dbname).C("system.profile").Count()
+}
+
+func isReplicasetEnabled(session pmgo.SessionManager) bool {
+	rss := proto.ReplicaSetStatus{}
+	if err := session.Run(bson.M{"replSetGetStatus": 1}, &rss); err != nil {
+		return false
+	}
+	return true
+}

src/go/pt-mongodb-query-digest/main_test.go

@@ -10,7 +10,9 @@ import (
 	"time"
 
 	"github.com/percona/percona-toolkit/src/go/mongolib/proto"
+	"github.com/percona/pmgo"
 
+	mgo "gopkg.in/mgo.v2"
 	"gopkg.in/mgo.v2/dbtest"
 )
 
@@ -50,7 +52,7 @@ func TestMain(m *testing.M) {
 
 func TestCalcStats(t *testing.T) {
 	it := Server.Session().DB("samples").C("system_profile").Find(nil).Sort("Ts").Iter()
-	data := getData(it)
+	data := getData(it, []docsFilter{})
 	s := calcStats(data[0].NScanned)
 
 	want := statistics{Pct: 0, Total: 159, Min: 79, Max: 80, Avg: 79.5, Pct95: 80, StdDev: 0.5, Median: 79.5}
@@ -117,26 +119,24 @@ func TestGetData(t *testing.T) {
 			want: []stat{
 				stat{
 					ID:             "6c3fff4804febd156700a06f9a346162",
+					Operation:      "query",
 					Fingerprint:    "find,limit",
 					Namespace:      "samples.col1",
-					Query: map[string]interface{}{
+					Query:          map[string]interface{}{"find": "col1", "limit": float64(2)},
-						"find":  "col1",
-						"limit": float64(2),
-					},
 					Count:          2,
 					TableScan:      false,
 					NScanned:       []float64{79, 80},
 					NReturned:      []float64{79, 80},
 					QueryTime:      []float64{27, 28},
 					ResponseLength: []float64{109, 110},
-					LockTime:       nil,
+					LockTime:       times(nil),
-					BlockedTime:    nil,
+					BlockedTime:    times(nil),
 					FirstSeen:      time.Date(2016, time.November, 8, 13, 46, 27, 0, time.UTC).Local(),
 					LastSeen:       time.Date(2016, time.November, 8, 13, 46, 27, 0, time.UTC).Local(),
 				},
-
 				stat{
 					ID:             "fdcea004122ddb225bc56de417391e25",
+					Operation:      "query",
 					Fingerprint:    "find",
 					Namespace:      "samples.col1",
 					Query:          map[string]interface{}{"find": "col1"},
@@ -146,8 +146,8 @@ func TestGetData(t *testing.T) {
 					NReturned:      []float64{71, 72, 73, 74, 75, 76, 77, 78},
 					QueryTime:      []float64{19, 20, 21, 22, 23, 24, 25, 26},
 					ResponseLength: []float64{101, 102, 103, 104, 105, 106, 107, 108},
-					LockTime:       nil,
+					LockTime:       times(nil),
-					BlockedTime:    nil,
+					BlockedTime:    times(nil),
 					FirstSeen:      time.Date(2016, time.November, 8, 13, 46, 27, 0, time.UTC).Local(),
 					LastSeen:       time.Date(2016, time.November, 8, 13, 46, 27, 0, time.UTC).Local(),
 				},
@@ -157,17 +157,17 @@ func TestGetData(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got := getData(tt.i)
+			got := getData(tt.i, []docsFilter{})
 			if !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("got\n%#v\nwant\n%#v", got, tt.want)
-
 			}
 		})
 	}
 }
 
 func TestUptime(t *testing.T) {
-	session := Server.Session()
+
+	session := pmgo.NewSessionManager(Server.Session())
 	time.Sleep(1500 * time.Millisecond)
 	if uptime(session) <= 0 {
 		t.Error("uptime is 0")
@@ -288,3 +288,22 @@ func TestTimesLess(t *testing.T) {
 		})
 	}
 }
+
+func TestIsProfilerEnabled(t *testing.T) {
+	mongoDSN := os.Getenv("PT_TEST_MONGODB_DSN")
+	if mongoDSN == "" {
+		t.Skip("Skippping TestIsProfilerEnabled. It runs only in integration tests")
+	}
+
+	dialer := pmgo.NewDialer()
+	di, _ := mgo.ParseURL(mongoDSN)
+	enabled, err := isProfilerEnabled(dialer, di)
+
+	if err != nil {
+		t.Errorf("Cannot check if profiler is enabled: %s", err.Error())
+	}
+	if enabled != true {
+		t.Error("Profiler must be enabled")
+	}
+
+}

src/go/pt-mongodb-query-digest/vendor/vendor.json

@@ -63,10 +63,10 @@
 			"revisionTime": "2017-02-01T15:06:01Z"
 		},
 		{
-			"checksumSHA1": "NviDzeLQRdyp9Ux2vMvu4zQvyYo=",
+			"checksumSHA1": "rDaYK9qS/RbE7w8jiPktdA3+lc0=",
 			"path": "github.com/percona/pmgo",
-			"revision": "2650f7f1545746eddae964e7308440900684c21a",
+			"revision": "9eabc5fda168f8d9961a56a767d8304e1b65135b",
-			"revisionTime": "2017-02-10T14:26:46Z"
+			"revisionTime": "2017-02-14T09:40:05Z"
 		},
 		{
 			"checksumSHA1": "ynJSWoF6v+3zMnh9R0QmmG6iGV8=",

src/go/pt-mongodb-summary/main.go

@@ -787,13 +787,13 @@ func parseFlags() options {
 
 	getopt.BoolVarLong(&opts.Help, "help", 'h', "Show help")
 	getopt.BoolVarLong(&opts.Version, "version", 'v', "", "Show version & exit")
-	getopt.BoolVarLong(&opts.NoVersionCheck, "no-version-check", 'c', "", "Don't check for updates")
+	getopt.BoolVarLong(&opts.NoVersionCheck, "no-version-check", 'c', "", "Default: Don't check for updates")
 
-	getopt.StringVarLong(&opts.User, "user", 'u', "", "User name")
+	getopt.StringVarLong(&opts.User, "username", 'u', "", "Username to use for optional MongoDB authentication")
-	getopt.StringVarLong(&opts.Password, "password", 'p', "", "Password").SetOptional()
+	getopt.StringVarLong(&opts.Password, "password", 'p', "", "Password to use for optional MongoDB authentication").SetOptional()
 	getopt.StringVarLong(&opts.AuthDB, "authenticationDatabase", 'a', "admin",
-		"Database used to establish credentials and privileges with a MongoDB server")
+		"Databaase to use for optional MongoDB authentication. Default: admin")
-	getopt.StringVarLong(&opts.LogLevel, "log-level", 'l', "error", "Log level:, panic, fatal, error, warn, info, debug")
+	getopt.StringVarLong(&opts.LogLevel, "log-level", 'l', "error", "Log level: panic, fatal, error, warn, info, debug. Default: error")
 
 	getopt.IntVarLong(&opts.RunningOpsSamples, "running-ops-samples", 's',
 		fmt.Sprintf("Number of samples to collect for running ops. Default: %d", opts.RunningOpsSamples))

src/go/pt-mongodb-summary/vendor/vendor.json

@@ -63,10 +63,10 @@
 			"revisionTime": "2017-02-01T15:06:01Z"
 		},
 		{
-			"checksumSHA1": "NviDzeLQRdyp9Ux2vMvu4zQvyYo=",
+			"checksumSHA1": "rDaYK9qS/RbE7w8jiPktdA3+lc0=",
 			"path": "github.com/percona/pmgo",
-			"revision": "2650f7f1545746eddae964e7308440900684c21a",
+			"revision": "9eabc5fda168f8d9961a56a767d8304e1b65135b",
-			"revisionTime": "2017-02-10T14:26:46Z"
+			"revisionTime": "2017-02-14T09:40:05Z"
 		},
 		{
 			"checksumSHA1": "kTY8RZr5j26RNIR+F2SnMLeB7G8=",