From 1d6adb953d970171000e89c19fcd6ac5032ec642 Mon Sep 17 00:00:00 2001 From: Carlos Salguero Date: Thu, 16 Feb 2017 15:59:01 -0300 Subject: [PATCH] New fingeprint method --- src/go/glide.lock | 75 +++++++++ src/go/mongolib/util/util.go | 9 +- src/go/pt-mongodb-query-digest/main.go | 145 +++++++++++++++--- src/go/pt-mongodb-query-digest/main_test.go | 46 +++++- src/go/pt-mongodb-summary/main.go | 17 +- .../templates/clusterwide.go | 4 +- 6 files changed, 260 insertions(+), 36 deletions(-) create mode 100644 src/go/glide.lock diff --git a/src/go/glide.lock b/src/go/glide.lock new file mode 100644 index 00000000..9040818f --- /dev/null +++ b/src/go/glide.lock @@ -0,0 +1,75 @@ +hash: 2ff7c989fb0fde1375999fded74ae44e10be513a21416571f026390b679924e4 +updated: 2017-02-16T13:05:08.118382254-03:00 +imports: +- name: github.com/bradfitz/slice + version: d9036e2120b5ddfa53f3ebccd618c4af275f47da +- name: github.com/go-ole/go-ole + version: de8695c8edbf8236f30d6e1376e20b198a028d42 + subpackages: + - oleutil +- name: github.com/golang/mock + version: bd3c8e81be01eef76d4b503f5e687d2d1354d2d9 + subpackages: + - gomock +- name: github.com/hashicorp/go-version + version: 03c5bf6be031b6dd45afec16b1cf94fc8938bc77 +- name: github.com/howeyc/gopass + version: bf9dde6d0d2c004a008c27aaee91170c786f6db8 +- name: github.com/kr/pretty + version: cfb55aafdaf3ec08f0db22699ab822c50091b1c4 +- name: github.com/kr/text + version: 7cafcd837844e784b526369c9bce262804aebc60 +- name: github.com/montanaflynn/stats + version: eeaced052adbcfeea372c749c281099ed7fdaa38 +- name: github.com/pborman/getopt + version: 7148bc3a4c3008adfcab60cbebfd0576018f330b +- name: github.com/percona/pmgo + version: 27d979df6c6885ff16abe375aead061a86da6df8 + subpackages: + - pmgomock +- name: github.com/pkg/errors + version: 645ef00459ed84a119197bfb8d8205042c6df63d +- name: github.com/satori/go.uuid + version: 879c5887cd475cd7864858769793b2ceb0d44feb +- name: github.com/shirou/gopsutil + version: 70a1b78fe69202d93d6718fc9e3a4d6f81edfd58 + subpackages: + - cpu + - host + - internal/common + - mem + - net + - process +- name: github.com/shirou/w32 + version: bb4de0191aa41b5507caa14b0650cdbddcd9280b +- name: github.com/sirupsen/logrus + version: c078b1e43f58d563c74cebe63c85789e76ddb627 +- name: github.com/StackExchange/wmi + version: e542ed97d15e640bdc14b5c12162d59e8fc67324 +- name: go4.org + version: 7ce08ca145dbe0e66a127c447b80ee7914f3e4f9 + subpackages: + - reflectutil +- name: golang.org/x/crypto + version: 453249f01cfeb54c3d549ddb75ff152ca243f9d8 + subpackages: + - ssh/terminal +- name: golang.org/x/net + version: 61557ac0112b576429a0df080e1c2cef5dfbb642 + subpackages: + - context +- name: golang.org/x/sys + version: e24f485414aeafb646f6fca458b0bf869c0880a1 + subpackages: + - unix +- name: gopkg.in/mgo.v2 + version: 3f83fa5005286a7fe593b055f0d7771a7dce4655 + subpackages: + - bson + - dbtest + - internal/json + - internal/sasl + - internal/scram +- name: gopkg.in/tomb.v2 + version: d5d1b5820637886def9eef33e03a27a9f166942c +testImports: [] diff --git a/src/go/mongolib/util/util.go b/src/go/mongolib/util/util.go index b1dad6c8..3c724222 100644 --- a/src/go/mongolib/util/util.go +++ b/src/go/mongolib/util/util.go @@ -36,7 +36,7 @@ func GetReplicasetMembers(dialer pmgo.Dialer, di *mgo.DialInfo) ([]proto.Members m := proto.Members{ Name: hostname, } - m.StateStr = cmdOpts.Parsed.Sharding.ClusterRole + m.StateStr = strings.ToUpper(cmdOpts.Parsed.Sharding.ClusterRole) if serverStatus, err := GetServerStatus(dialer, di, m.Name); err == nil { m.ID = serverStatus.Pid @@ -54,7 +54,12 @@ func GetReplicasetMembers(dialer pmgo.Dialer, di *mgo.DialInfo) ([]proto.Members if serverStatus, err := GetServerStatus(dialer, di, m.Name); err == nil { m.ID = serverStatus.Pid m.StorageEngine = serverStatus.StorageEngine - m.StateStr = cmdOpts.Parsed.Sharding.ClusterRole + "/" + m.StateStr + if cmdOpts.Parsed.Sharding.ClusterRole == "" { + m.StateStr = m.StateStr + } else { + m.StateStr = cmdOpts.Parsed.Sharding.ClusterRole + "/" + m.StateStr + } + m.StateStr = strings.ToUpper(m.StateStr) } membersMap[m.Name] = m } diff --git a/src/go/pt-mongodb-query-digest/main.go b/src/go/pt-mongodb-query-digest/main.go index 12001f99..a760dbed 100644 --- a/src/go/pt-mongodb-query-digest/main.go +++ b/src/go/pt-mongodb-query-digest/main.go @@ -3,8 +3,10 @@ package main import ( "crypto/md5" "encoding/json" + "errors" "fmt" "os" + "regexp" "sort" "strings" "text/template" @@ -27,12 +29,25 @@ import ( const ( TOOLNAME = "pt-mongodb-query-digest" MAX_DEPTH_LEVEL = 10 + + DEFAULT_AUTHDB = "admin" + DEFAULT_HOST = "localhost:27017" + DEFAULT_LOGLEVEL = "warn" + DEFAULT_ORDERBY = "count" // comma separated list + DEFAULT_SKIPCOLLECTIONS = "system.profile" // comma separated list ) var ( - Version string - Build string - GoVersion string + Build string = "01-01-1980" + GoVersion string = "1.8" + Version string = "3.0.1" + + CANNOT_GET_QUERY_ERROR = errors.New("cannot get query field from the profile document (it is not a map)") + + // This is a regexp array to filter out the keys we don't want in the fingerprint + keyFilters = func() []string { + return []string{"^shardVersion$", "^\\$"} + } ) type iter interface { @@ -433,13 +448,12 @@ func getData(i iter, filters []docsFilter) []stat { log.Debugln("====================================================================================================") log.Debug(pretty.Sprint(doc)) if len(doc.Query) > 0 { - query := doc.Query - if squery, ok := doc.Query["$query"]; ok { - if ssquery, ok := squery.(map[string]interface{}); ok { - query = ssquery - } + + fp, err := fingerprint(doc.Query) + if err != nil { + log.Errorf("cannot get fingerprint: %s", err.Error()) + continue } - fp := fingerprint(query) var s *stat var ok bool key := groupKey{ @@ -448,13 +462,14 @@ func getData(i iter, filters []docsFilter) []stat { Namespace: doc.Ns, } if s, ok = stats[key]; !ok { + realQuery, _ := getQueryField(doc.Query) s = &stat{ ID: fmt.Sprintf("%x", md5.Sum([]byte(fp+doc.Ns))), Operation: doc.Op, Fingerprint: fp, Namespace: doc.Ns, TableScan: false, - Query: query, + Query: realQuery, } stats[key] = s } @@ -486,10 +501,11 @@ func getData(i iter, filters []docsFilter) []stat { func getOptions() (*options, error) { opts := &options{ - Host: "localhost:27017", - LogLevel: "warn", - OrderBy: []string{"count"}, - SkipCollections: []string{"system.profile"}, + Host: DEFAULT_HOST, + LogLevel: DEFAULT_LOGLEVEL, + OrderBy: strings.Split(DEFAULT_ORDERBY, ","), + SkipCollections: strings.Split(DEFAULT_SKIPCOLLECTIONS, ","), + AuthDB: DEFAULT_AUTHDB, } getopt.BoolVarLong(&opts.Help, "help", '?', "Show help") @@ -573,14 +589,83 @@ func getDialInfo(opts *options) *mgo.DialInfo { return di } -func fingerprint(query map[string]interface{}) string { - return strings.Join(keys(query, 0), ",") +func getQueryField(query map[string]interface{}) (map[string]interface{}, error) { + // MongoDB 3.0 + if squery, ok := query["$query"]; ok { + // just an extra check to ensure this type assertion won't fail + if ssquery, ok := squery.(map[string]interface{}); ok { + return ssquery, nil + } + return nil, CANNOT_GET_QUERY_ERROR + } + // MongoDB 3.2+ + if squery, ok := query["filter"]; ok { + if ssquery, ok := squery.(map[string]interface{}); ok { + return ssquery, nil + } + return nil, CANNOT_GET_QUERY_ERROR + } + return query, nil +} + +// Query is the top level map query element +// Example for MongoDB 3.2+ +// "query" : { +// "find" : "col1", +// "filter" : { +// "s2" : { +// "$lt" : "54701", +// "$gte" : "73754" +// } +// }, +// "sort" : { +// "user_id" : 1 +// } +// } +func fingerprint(query map[string]interface{}) (string, error) { + + realQuery, err := getQueryField(query) + if err != nil { + // Try to encode doc.Query as json for prettiness + if buf, err := json.Marshal(realQuery); err == nil { + return "", fmt.Errorf("%v for query %s", err, string(buf)) + } + // If we cannot encode as json, return just the error message without the query + return "", err + } + retKeys := keys(realQuery, 0) + + sort.Strings(retKeys) + + // if there is a sort clause in the query, we have to add all fields in the sort + // fields list that are not in the query keys list (retKeys) + if sortKeys, ok := query["sort"]; ok { + if sortKeysMap, ok := sortKeys.(map[string]interface{}); ok { + sortKeys := mapKeys(sortKeysMap, 0) + for _, sortKey := range sortKeys { + if !inSlice(sortKey, retKeys) { + retKeys = append(retKeys, sortKey) + } + } + } + } + + return strings.Join(retKeys, ","), nil +} + +func inSlice(str string, list []string) bool { + for _, v := range list { + if v == str { + return true + } + } + return false } func keys(query map[string]interface{}, level int) []string { ks := []string{} for key, value := range query { - if !shouldIncludeKey(key) { + if shouldSkipKey(key) { continue } ks = append(ks, key) @@ -595,14 +680,28 @@ func keys(query map[string]interface{}, level int) []string { return ks } -func shouldIncludeKey(key string) bool { - filterOut := []string{"shardVersion"} - for _, val := range filterOut { - if val == key { - return false +func mapKeys(query map[string]interface{}, level int) []string { + ks := []string{} + for key, value := range query { + ks = append(ks, key) + if m, ok := value.(map[string]interface{}); ok { + level++ + if level <= MAX_DEPTH_LEVEL { + ks = append(ks, keys(m, level)...) + } } } - return true + sort.Strings(ks) + return ks +} + +func shouldSkipKey(key string) bool { + for _, filter := range keyFilters() { + if matched, _ := regexp.MatchString(filter, key); matched { + return true + } + } + return false } func printHeader(opts *options) { diff --git a/src/go/pt-mongodb-query-digest/main_test.go b/src/go/pt-mongodb-query-digest/main_test.go index 12e9bff9..2ec5fad5 100644 --- a/src/go/pt-mongodb-query-digest/main_test.go +++ b/src/go/pt-mongodb-query-digest/main_test.go @@ -188,18 +188,56 @@ func TestFingerprint(t *testing.T) { }, { query: map[string]interface{}{"find": "system.profile", "filter": map[string]interface{}{}, "sort": map[string]interface{}{"$natural": 1}}, - want: "$natural,filter,find,sort", + want: "$natural", }, { query: map[string]interface{}{"collection": "system.profile", "batchSize": 0, "getMore": 18531768265}, want: "batchSize,collection,getMore", }, + /* + Main test case: + Got Query field: + { + "filter": { + "latestFeedbackDate":{ + "$gte":1427846400000, + "$lte":1486511999999}, + "merchantId":"560bc82a498e0b791959be71", + "reviewed":true, + "serviceFeedback.fiveStarScore.selectedScore":{ + "$in":[5,4,3,2,1] + } + }, + "find": "saleUpdatedTags", + "ntoreturn":10, + "projection":{ + "$sortKey":{ + "$meta":"sortKey" + } + }, + "shardVersion":[571230652140,"55d1b3f1e6845ce25be7e6db"], + "sort":{"latestFeedbackDate":-1} + } + + Want fingerprint: + latestFeedbackDate,merchantId,reviewed,serviceFeedback.fiveStarScore.selectedScore + + Why? + 1) It is MongoDb 3.2+ (has filter instead of $query) + 2) From the "filter" map, we are removing all keys starting with $ + 3) The key 'latestFeedbackDate' exists in the "sort" map but it is not in the "filter" keys + so it has been added to the final fingerprint + */ + { + query: map[string]interface{}{"sort": map[string]interface{}{"latestFeedbackDate": -1}, "filter": map[string]interface{}{"latestFeedbackDate": map[string]interface{}{"$gte": 1.4278464e+12, "$lte": 1.486511999999e+12}, "merchantId": "560bc82a498e0b791959be71", "reviewed": true, "serviceFeedback.fiveStarScore.selectedScore": map[string]interface{}{"$in": []interface{}{5, 4, 3, 2, 1}}}, "find": "saleUpdatedTags", "ntoreturn": 10, "projection": map[string]interface{}{"$sortKey": map[string]interface{}{"$meta": "sortKey"}}, "shardVersion": []interface{}{5.7123065214e+11, "55d1b3f1e6845ce25be7e6db"}}, + want: "latestFeedbackDate,merchantId,reviewed,serviceFeedback.fiveStarScore.selectedScore", + }, } - for _, tt := range tests { + for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := fingerprint(tt.query); got != tt.want { - t.Errorf("fingerprint() = %v, want %v", got, tt.want) + if got, err := fingerprint(tt.query); got != tt.want || err != nil { + t.Errorf("fingerprint case #%d:\n got %v,\nwant %v\nerror: %v\n", i, got, tt.want, err) } }) } diff --git a/src/go/pt-mongodb-summary/main.go b/src/go/pt-mongodb-summary/main.go index 7d7aad25..1b9394af 100644 --- a/src/go/pt-mongodb-summary/main.go +++ b/src/go/pt-mongodb-summary/main.go @@ -27,12 +27,18 @@ import ( const ( TOOLNAME = "pt-mongodb-summary" + + DEFAULT_AUTHDB = "admin" + DEFAULT_HOST = "localhost:27017" + DEFAULT_LOGLEVEL = "warn" + DEFAULT_RUNNINGOPSINTERVAL = 1000 // milliseconds + DEFAULT_RUNNINGOPSSAMPLES = 5 ) var ( - Version string = "2.2.19" Build string = "01-01-1980" GoVersion string = "1.8" + Version string = "3.0.1" ) type TimedStats struct { @@ -779,10 +785,11 @@ func externalIP() (string, error) { func parseFlags() options { opts := options{ - Host: "localhost:27017", - LogLevel: "warn", - RunningOpsSamples: 5, - RunningOpsInterval: 1000, // milliseconds + Host: DEFAULT_HOST, + LogLevel: DEFAULT_LOGLEVEL, + RunningOpsSamples: DEFAULT_RUNNINGOPSSAMPLES, + RunningOpsInterval: DEFAULT_RUNNINGOPSINTERVAL, // milliseconds + AuthDB: DEFAULT_AUTHDB, } getopt.BoolVarLong(&opts.Help, "help", 'h', "Show help") diff --git a/src/go/pt-mongodb-summary/templates/clusterwide.go b/src/go/pt-mongodb-summary/templates/clusterwide.go index aa0b3427..06aa737a 100644 --- a/src/go/pt-mongodb-summary/templates/clusterwide.go +++ b/src/go/pt-mongodb-summary/templates/clusterwide.go @@ -10,8 +10,8 @@ Unsharded Collections: {{.UnshardedColsCount}} Unsharded Data Size: {{.UnshardedDataSizeScaled}} {{.UnshardedDataSizeScale}} {{- if .Chunks }} ### Chunks: - {{- range .Chunks }} - {{ printf "%30s" .ID}}: {{ printf "%5d" .Count }} + {{ range .Chunks }} +{{- if .ID }} {{ printf "%5d" .Count }} : {{ printf "%-30s" .ID}}{{ end -}} {{- end -}} {{ end -}} `