New fingeprint method

src/go/glide.lock

@@ -0,0 +1,75 @@
+hash: 2ff7c989fb0fde1375999fded74ae44e10be513a21416571f026390b679924e4
+updated: 2017-02-16T13:05:08.118382254-03:00
+imports:
+- name: github.com/bradfitz/slice
+  version: d9036e2120b5ddfa53f3ebccd618c4af275f47da
+- name: github.com/go-ole/go-ole
+  version: de8695c8edbf8236f30d6e1376e20b198a028d42
+  subpackages:
+  - oleutil
+- name: github.com/golang/mock
+  version: bd3c8e81be01eef76d4b503f5e687d2d1354d2d9
+  subpackages:
+  - gomock
+- name: github.com/hashicorp/go-version
+  version: 03c5bf6be031b6dd45afec16b1cf94fc8938bc77
+- name: github.com/howeyc/gopass
+  version: bf9dde6d0d2c004a008c27aaee91170c786f6db8
+- name: github.com/kr/pretty
+  version: cfb55aafdaf3ec08f0db22699ab822c50091b1c4
+- name: github.com/kr/text
+  version: 7cafcd837844e784b526369c9bce262804aebc60
+- name: github.com/montanaflynn/stats
+  version: eeaced052adbcfeea372c749c281099ed7fdaa38
+- name: github.com/pborman/getopt
+  version: 7148bc3a4c3008adfcab60cbebfd0576018f330b
+- name: github.com/percona/pmgo
+  version: 27d979df6c6885ff16abe375aead061a86da6df8
+  subpackages:
+  - pmgomock
+- name: github.com/pkg/errors
+  version: 645ef00459ed84a119197bfb8d8205042c6df63d
+- name: github.com/satori/go.uuid
+  version: 879c5887cd475cd7864858769793b2ceb0d44feb
+- name: github.com/shirou/gopsutil
+  version: 70a1b78fe69202d93d6718fc9e3a4d6f81edfd58
+  subpackages:
+  - cpu
+  - host
+  - internal/common
+  - mem
+  - net
+  - process
+- name: github.com/shirou/w32
+  version: bb4de0191aa41b5507caa14b0650cdbddcd9280b
+- name: github.com/sirupsen/logrus
+  version: c078b1e43f58d563c74cebe63c85789e76ddb627
+- name: github.com/StackExchange/wmi
+  version: e542ed97d15e640bdc14b5c12162d59e8fc67324
+- name: go4.org
+  version: 7ce08ca145dbe0e66a127c447b80ee7914f3e4f9
+  subpackages:
+  - reflectutil
+- name: golang.org/x/crypto
+  version: 453249f01cfeb54c3d549ddb75ff152ca243f9d8
+  subpackages:
+  - ssh/terminal
+- name: golang.org/x/net
+  version: 61557ac0112b576429a0df080e1c2cef5dfbb642
+  subpackages:
+  - context
+- name: golang.org/x/sys
+  version: e24f485414aeafb646f6fca458b0bf869c0880a1
+  subpackages:
+  - unix
+- name: gopkg.in/mgo.v2
+  version: 3f83fa5005286a7fe593b055f0d7771a7dce4655
+  subpackages:
+  - bson
+  - dbtest
+  - internal/json
+  - internal/sasl
+  - internal/scram
+- name: gopkg.in/tomb.v2
+  version: d5d1b5820637886def9eef33e03a27a9f166942c
+testImports: []

src/go/mongolib/util/util.go

@@ -36,7 +36,7 @@ func GetReplicasetMembers(dialer pmgo.Dialer, di *mgo.DialInfo) ([]proto.Members
 			m := proto.Members{
 				Name: hostname,
 			}
-			m.StateStr = cmdOpts.Parsed.Sharding.ClusterRole
+			m.StateStr = strings.ToUpper(cmdOpts.Parsed.Sharding.ClusterRole)
 
 			if serverStatus, err := GetServerStatus(dialer, di, m.Name); err == nil {
 				m.ID = serverStatus.Pid
@@ -54,8 +54,13 @@ func GetReplicasetMembers(dialer pmgo.Dialer, di *mgo.DialInfo) ([]proto.Members
 			if serverStatus, err := GetServerStatus(dialer, di, m.Name); err == nil {
 				m.ID = serverStatus.Pid
 				m.StorageEngine = serverStatus.StorageEngine
+				if cmdOpts.Parsed.Sharding.ClusterRole == "" {
+					m.StateStr = m.StateStr
+				} else {
 					m.StateStr = cmdOpts.Parsed.Sharding.ClusterRole + "/" + m.StateStr
 				}
+				m.StateStr = strings.ToUpper(m.StateStr)
+			}
 			membersMap[m.Name] = m
 		}
 

src/go/pt-mongodb-query-digest/main.go

@@ -3,8 +3,10 @@ package main
 import (
 	"crypto/md5"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
+	"regexp"
 	"sort"
 	"strings"
 	"text/template"
@@ -27,12 +29,25 @@ import (
 const (
 	TOOLNAME        = "pt-mongodb-query-digest"
 	MAX_DEPTH_LEVEL = 10
+
+	DEFAULT_AUTHDB          = "admin"
+	DEFAULT_HOST            = "localhost:27017"
+	DEFAULT_LOGLEVEL        = "warn"
+	DEFAULT_ORDERBY         = "count"          // comma separated list
+	DEFAULT_SKIPCOLLECTIONS = "system.profile" // comma separated list
 )
 
 var (
-	Version   string
+	Build     string = "01-01-1980"
-	Build     string
+	GoVersion string = "1.8"
-	GoVersion string
+	Version   string = "3.0.1"
+
+	CANNOT_GET_QUERY_ERROR = errors.New("cannot get query field from the profile document (it is not a map)")
+
+	// This is a regexp array to filter out the keys we don't want in the fingerprint
+	keyFilters = func() []string {
+		return []string{"^shardVersion$", "^\\$"}
+	}
 )
 
 type iter interface {
@@ -433,13 +448,12 @@ func getData(i iter, filters []docsFilter) []stat {
 		log.Debugln("====================================================================================================")
 		log.Debug(pretty.Sprint(doc))
 		if len(doc.Query) > 0 {
-			query := doc.Query
+
-			if squery, ok := doc.Query["$query"]; ok {
+			fp, err := fingerprint(doc.Query)
-				if ssquery, ok := squery.(map[string]interface{}); ok {
+			if err != nil {
-					query = ssquery
+				log.Errorf("cannot get fingerprint: %s", err.Error())
+				continue
 			}
-			}
-			fp := fingerprint(query)
 			var s *stat
 			var ok bool
 			key := groupKey{
@@ -448,13 +462,14 @@ func getData(i iter, filters []docsFilter) []stat {
 				Namespace:   doc.Ns,
 			}
 			if s, ok = stats[key]; !ok {
+				realQuery, _ := getQueryField(doc.Query)
 				s = &stat{
 					ID:          fmt.Sprintf("%x", md5.Sum([]byte(fp+doc.Ns))),
 					Operation:   doc.Op,
 					Fingerprint: fp,
 					Namespace:   doc.Ns,
 					TableScan:   false,
-					Query:       query,
+					Query:       realQuery,
 				}
 				stats[key] = s
 			}
@@ -486,10 +501,11 @@ func getData(i iter, filters []docsFilter) []stat {
 
 func getOptions() (*options, error) {
 	opts := &options{
-		Host:            "localhost:27017",
+		Host:            DEFAULT_HOST,
-		LogLevel:        "warn",
+		LogLevel:        DEFAULT_LOGLEVEL,
-		OrderBy:         []string{"count"},
+		OrderBy:         strings.Split(DEFAULT_ORDERBY, ","),
-		SkipCollections: []string{"system.profile"},
+		SkipCollections: strings.Split(DEFAULT_SKIPCOLLECTIONS, ","),
+		AuthDB:          DEFAULT_AUTHDB,
 	}
 
 	getopt.BoolVarLong(&opts.Help, "help", '?', "Show help")
@@ -573,14 +589,83 @@ func getDialInfo(opts *options) *mgo.DialInfo {
 	return di
 }
 
-func fingerprint(query map[string]interface{}) string {
+func getQueryField(query map[string]interface{}) (map[string]interface{}, error) {
-	return strings.Join(keys(query, 0), ",")
+	// MongoDB 3.0
+	if squery, ok := query["$query"]; ok {
+		// just an extra check to ensure this type assertion won't fail
+		if ssquery, ok := squery.(map[string]interface{}); ok {
+			return ssquery, nil
+		}
+		return nil, CANNOT_GET_QUERY_ERROR
+	}
+	// MongoDB 3.2+
+	if squery, ok := query["filter"]; ok {
+		if ssquery, ok := squery.(map[string]interface{}); ok {
+			return ssquery, nil
+		}
+		return nil, CANNOT_GET_QUERY_ERROR
+	}
+	return query, nil
+}
+
+// Query is the top level map query element
+// Example for MongoDB 3.2+
+//     "query" : {
+//        "find" : "col1",
+//        "filter" : {
+//            "s2" : {
+//                "$lt" : "54701",
+//                "$gte" : "73754"
+//            }
+//        },
+//        "sort" : {
+//            "user_id" : 1
+//        }
+//     }
+func fingerprint(query map[string]interface{}) (string, error) {
+
+	realQuery, err := getQueryField(query)
+	if err != nil {
+		// Try to encode doc.Query as json for prettiness
+		if buf, err := json.Marshal(realQuery); err == nil {
+			return "", fmt.Errorf("%v for query %s", err, string(buf))
+		}
+		// If we cannot encode as json, return just the error message without the query
+		return "", err
+	}
+	retKeys := keys(realQuery, 0)
+
+	sort.Strings(retKeys)
+
+	// if there is a sort clause in the query, we have to add all fields in the sort
+	// fields list that are not in the query keys list (retKeys)
+	if sortKeys, ok := query["sort"]; ok {
+		if sortKeysMap, ok := sortKeys.(map[string]interface{}); ok {
+			sortKeys := mapKeys(sortKeysMap, 0)
+			for _, sortKey := range sortKeys {
+				if !inSlice(sortKey, retKeys) {
+					retKeys = append(retKeys, sortKey)
+				}
+			}
+		}
+	}
+
+	return strings.Join(retKeys, ","), nil
+}
+
+func inSlice(str string, list []string) bool {
+	for _, v := range list {
+		if v == str {
+			return true
+		}
+	}
+	return false
 }
 
 func keys(query map[string]interface{}, level int) []string {
 	ks := []string{}
 	for key, value := range query {
-		if !shouldIncludeKey(key) {
+		if shouldSkipKey(key) {
 			continue
 		}
 		ks = append(ks, key)
@@ -595,15 +680,29 @@ func keys(query map[string]interface{}, level int) []string {
 	return ks
 }
 
-func shouldIncludeKey(key string) bool {
+func mapKeys(query map[string]interface{}, level int) []string {
-	filterOut := []string{"shardVersion"}
+	ks := []string{}
-	for _, val := range filterOut {
+	for key, value := range query {
-		if val == key {
+		ks = append(ks, key)
-			return false
+		if m, ok := value.(map[string]interface{}); ok {
+			level++
+			if level <= MAX_DEPTH_LEVEL {
+				ks = append(ks, keys(m, level)...)
 			}
 		}
+	}
+	sort.Strings(ks)
+	return ks
+}
+
+func shouldSkipKey(key string) bool {
+	for _, filter := range keyFilters() {
+		if matched, _ := regexp.MatchString(filter, key); matched {
 			return true
 		}
+	}
+	return false
+}
 
 func printHeader(opts *options) {
 	fmt.Printf("%s - %s\n", TOOLNAME, time.Now().Format(time.RFC1123Z))

src/go/pt-mongodb-query-digest/main_test.go

@@ -188,18 +188,56 @@ func TestFingerprint(t *testing.T) {
 		},
 		{
 			query: map[string]interface{}{"find": "system.profile", "filter": map[string]interface{}{}, "sort": map[string]interface{}{"$natural": 1}},
-			want:  "$natural,filter,find,sort",
+			want:  "$natural",
 		},
 		{
 
 			query: map[string]interface{}{"collection": "system.profile", "batchSize": 0, "getMore": 18531768265},
 			want:  "batchSize,collection,getMore",
 		},
+		/*
+			  Main test case:
+			  Got Query field:
+			  {
+				  "filter": {
+				     "latestFeedbackDate":{
+				        "$gte":1427846400000,
+				        "$lte":1486511999999},
+				        "merchantId":"560bc82a498e0b791959be71",
+				        "reviewed":true,
+				        "serviceFeedback.fiveStarScore.selectedScore":{
+				            "$in":[5,4,3,2,1]
 				        }
-	for _, tt := range tests {
+				  },
+				  "find": "saleUpdatedTags",
+				  "ntoreturn":10,
+				  "projection":{
+				     "$sortKey":{
+				        "$meta":"sortKey"
+				     }
+				  },
+				  "shardVersion":[571230652140,"55d1b3f1e6845ce25be7e6db"],
+				  "sort":{"latestFeedbackDate":-1}
+			  }
+
+			  Want fingerprint:
+			  latestFeedbackDate,merchantId,reviewed,serviceFeedback.fiveStarScore.selectedScore
+
+			  Why?
+			  1) It is MongoDb 3.2+ (has filter instead of $query)
+			  2) From the "filter" map, we are removing all keys starting with $
+			  3) The key 'latestFeedbackDate' exists in the "sort" map but it is not in the "filter" keys
+			     so it has been added to the final fingerprint
+		*/
+		{
+			query: map[string]interface{}{"sort": map[string]interface{}{"latestFeedbackDate": -1}, "filter": map[string]interface{}{"latestFeedbackDate": map[string]interface{}{"$gte": 1.4278464e+12, "$lte": 1.486511999999e+12}, "merchantId": "560bc82a498e0b791959be71", "reviewed": true, "serviceFeedback.fiveStarScore.selectedScore": map[string]interface{}{"$in": []interface{}{5, 4, 3, 2, 1}}}, "find": "saleUpdatedTags", "ntoreturn": 10, "projection": map[string]interface{}{"$sortKey": map[string]interface{}{"$meta": "sortKey"}}, "shardVersion": []interface{}{5.7123065214e+11, "55d1b3f1e6845ce25be7e6db"}},
+			want:  "latestFeedbackDate,merchantId,reviewed,serviceFeedback.fiveStarScore.selectedScore",
+		},
+	}
+	for i, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := fingerprint(tt.query); got != tt.want {
+			if got, err := fingerprint(tt.query); got != tt.want || err != nil {
-				t.Errorf("fingerprint() = %v, want %v", got, tt.want)
+				t.Errorf("fingerprint  case #%d:\n got %v,\nwant %v\nerror: %v\n", i, got, tt.want, err)
 			}
 		})
 	}

src/go/pt-mongodb-summary/main.go

@@ -27,12 +27,18 @@ import (
 
 const (
 	TOOLNAME = "pt-mongodb-summary"
+
+	DEFAULT_AUTHDB             = "admin"
+	DEFAULT_HOST               = "localhost:27017"
+	DEFAULT_LOGLEVEL           = "warn"
+	DEFAULT_RUNNINGOPSINTERVAL = 1000 // milliseconds
+	DEFAULT_RUNNINGOPSSAMPLES  = 5
 )
 
 var (
-	Version   string = "2.2.19"
 	Build     string = "01-01-1980"
 	GoVersion string = "1.8"
+	Version   string = "3.0.1"
 )
 
 type TimedStats struct {
@@ -779,10 +785,11 @@ func externalIP() (string, error) {
 
 func parseFlags() options {
 	opts := options{
-		Host:               "localhost:27017",
+		Host:               DEFAULT_HOST,
-		LogLevel:           "warn",
+		LogLevel:           DEFAULT_LOGLEVEL,
-		RunningOpsSamples:  5,
+		RunningOpsSamples:  DEFAULT_RUNNINGOPSSAMPLES,
-		RunningOpsInterval: 1000, // milliseconds
+		RunningOpsInterval: DEFAULT_RUNNINGOPSINTERVAL, // milliseconds
+		AuthDB:             DEFAULT_AUTHDB,
 	}
 
 	getopt.BoolVarLong(&opts.Help, "help", 'h', "Show help")

src/go/pt-mongodb-summary/templates/clusterwide.go

@@ -10,8 +10,8 @@ Unsharded Collections: {{.UnshardedColsCount}}
   Unsharded Data Size: {{.UnshardedDataSizeScaled}} {{.UnshardedDataSizeScale}}
 {{- if .Chunks }}
           ###  Chunks: 
-	{{- range .Chunks }}
+	{{ range .Chunks }}
-         {{ printf "%30s" .ID}}: {{ printf "%5d" .Count }}
+{{- if .ID }}       {{ printf "%5d" .Count }} : {{ printf "%-30s" .ID}}{{ end -}}
 	{{- end -}}
 {{ end -}}
 `