percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-10 05:00:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

New fingeprint method

Browse Source
This commit is contained in:
Carlos Salguero
2017-02-16 15:59:01 -03:00
parent 28a1870de9
commit 1d6adb953d
6 changed files with 260 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
src/go/glide.lock generated Normal file
View File

@@ -0,0 +1,75 @@
hash: 2ff7c989fb0fde1375999fded74ae44e10be513a21416571f026390b679924e4
updated: 2017-02-16T13:05:08.118382254-03:00
imports:
- name: github.com/bradfitz/slice
version: d9036e2120b5ddfa53f3ebccd618c4af275f47da
- name: github.com/go-ole/go-ole
version: de8695c8edbf8236f30d6e1376e20b198a028d42
subpackages:
- oleutil
- name: github.com/golang/mock
version: bd3c8e81be01eef76d4b503f5e687d2d1354d2d9
subpackages:
- gomock
- name: github.com/hashicorp/go-version
version: 03c5bf6be031b6dd45afec16b1cf94fc8938bc77
- name: github.com/howeyc/gopass
version: bf9dde6d0d2c004a008c27aaee91170c786f6db8
- name: github.com/kr/pretty
version: cfb55aafdaf3ec08f0db22699ab822c50091b1c4
- name: github.com/kr/text
version: 7cafcd837844e784b526369c9bce262804aebc60
- name: github.com/montanaflynn/stats
version: eeaced052adbcfeea372c749c281099ed7fdaa38
- name: github.com/pborman/getopt
version: 7148bc3a4c3008adfcab60cbebfd0576018f330b
- name: github.com/percona/pmgo
version: 27d979df6c6885ff16abe375aead061a86da6df8
subpackages:
- pmgomock
- name: github.com/pkg/errors
version: 645ef00459ed84a119197bfb8d8205042c6df63d
- name: github.com/satori/go.uuid
version: 879c5887cd475cd7864858769793b2ceb0d44feb
- name: github.com/shirou/gopsutil
version: 70a1b78fe69202d93d6718fc9e3a4d6f81edfd58
subpackages:
- cpu
- host
- internal/common
- mem
- net
- process
- name: github.com/shirou/w32
version: bb4de0191aa41b5507caa14b0650cdbddcd9280b
- name: github.com/sirupsen/logrus
version: c078b1e43f58d563c74cebe63c85789e76ddb627
- name: github.com/StackExchange/wmi
version: e542ed97d15e640bdc14b5c12162d59e8fc67324
- name: go4.org
version: 7ce08ca145dbe0e66a127c447b80ee7914f3e4f9
subpackages:
- reflectutil
- name: golang.org/x/crypto
version: 453249f01cfeb54c3d549ddb75ff152ca243f9d8
subpackages:
- ssh/terminal
- name: golang.org/x/net
version: 61557ac0112b576429a0df080e1c2cef5dfbb642
subpackages:
- context
- name: golang.org/x/sys
version: e24f485414aeafb646f6fca458b0bf869c0880a1
subpackages:
- unix
- name: gopkg.in/mgo.v2
version: 3f83fa5005286a7fe593b055f0d7771a7dce4655
subpackages:
- bson
- dbtest
- internal/json
- internal/sasl
- internal/scram
- name: gopkg.in/tomb.v2
version: d5d1b5820637886def9eef33e03a27a9f166942c
testImports: []

9
src/go/mongolib/util/util.go
View File

@@ -36,7 +36,7 @@ func GetReplicasetMembers(dialer pmgo.Dialer, di *mgo.DialInfo) ([]proto.Members
m := proto.Members{
Name: hostname,
}
m.StateStr = cmdOpts.Parsed.Sharding.ClusterRole
m.StateStr = strings.ToUpper(cmdOpts.Parsed.Sharding.ClusterRole)
if serverStatus, err := GetServerStatus(dialer, di, m.Name); err == nil {
m.ID = serverStatus.Pid
@@ -54,7 +54,12 @@ func GetReplicasetMembers(dialer pmgo.Dialer, di *mgo.DialInfo) ([]proto.Members
if serverStatus, err := GetServerStatus(dialer, di, m.Name); err == nil {
m.ID = serverStatus.Pid
m.StorageEngine = serverStatus.StorageEngine
m.StateStr = cmdOpts.Parsed.Sharding.ClusterRole + "/" + m.StateStr
if cmdOpts.Parsed.Sharding.ClusterRole == "" {
m.StateStr = m.StateStr
} else {
m.StateStr = cmdOpts.Parsed.Sharding.ClusterRole + "/" + m.StateStr
}
m.StateStr = strings.ToUpper(m.StateStr)
}
membersMap[m.Name] = m
}

145
src/go/pt-mongodb-query-digest/main.go
View File

@@ -3,8 +3,10 @@ package main
import (
"crypto/md5"
"encoding/json"
"errors"
"fmt"
"os"
"regexp"
"sort"
"strings"
"text/template"
@@ -27,12 +29,25 @@ import (
const (
TOOLNAME = "pt-mongodb-query-digest"
MAX_DEPTH_LEVEL = 10
DEFAULT_AUTHDB = "admin"
DEFAULT_HOST = "localhost:27017"
DEFAULT_LOGLEVEL = "warn"
DEFAULT_ORDERBY = "count" // comma separated list
DEFAULT_SKIPCOLLECTIONS = "system.profile" // comma separated list
)
var (
Version string
Build string
GoVersion string
Build string = "01-01-1980"
GoVersion string = "1.8"
Version string = "3.0.1"
CANNOT_GET_QUERY_ERROR = errors.New("cannot get query field from the profile document (it is not a map)")
// This is a regexp array to filter out the keys we don't want in the fingerprint
keyFilters = func() []string {
return []string{"^shardVersion$", "^\\$"}
}
)
type iter interface {
@@ -433,13 +448,12 @@ func getData(i iter, filters []docsFilter) []stat {
log.Debugln("====================================================================================================")
log.Debug(pretty.Sprint(doc))
if len(doc.Query) > 0 {
query := doc.Query
if squery, ok := doc.Query["$query"]; ok {
if ssquery, ok := squery.(map[string]interface{}); ok {
query = ssquery
}
fp, err := fingerprint(doc.Query)
if err != nil {
log.Errorf("cannot get fingerprint: %s", err.Error())
continue
}
fp := fingerprint(query)
var s *stat
var ok bool
key := groupKey{
@@ -448,13 +462,14 @@ func getData(i iter, filters []docsFilter) []stat {
Namespace: doc.Ns,
}
if s, ok = stats[key]; !ok {
realQuery, _ := getQueryField(doc.Query)
s = &stat{
ID: fmt.Sprintf("%x", md5.Sum([]byte(fp+doc.Ns))),
Operation: doc.Op,
Fingerprint: fp,
Namespace: doc.Ns,
TableScan: false,
Query: query,
Query: realQuery,
}
stats[key] = s
}
@@ -486,10 +501,11 @@ func getData(i iter, filters []docsFilter) []stat {
func getOptions() (*options, error) {
opts := &options{
Host: "localhost:27017",
LogLevel: "warn",
OrderBy: []string{"count"},
SkipCollections: []string{"system.profile"},
Host: DEFAULT_HOST,
LogLevel: DEFAULT_LOGLEVEL,
OrderBy: strings.Split(DEFAULT_ORDERBY, ","),
SkipCollections: strings.Split(DEFAULT_SKIPCOLLECTIONS, ","),
AuthDB: DEFAULT_AUTHDB,
}
getopt.BoolVarLong(&opts.Help, "help", '?', "Show help")
@@ -573,14 +589,83 @@ func getDialInfo(opts *options) *mgo.DialInfo {
return di
}
func fingerprint(query map[string]interface{}) string {
return strings.Join(keys(query, 0), ",")
func getQueryField(query map[string]interface{}) (map[string]interface{}, error) {
// MongoDB 3.0
if squery, ok := query["$query"]; ok {
// just an extra check to ensure this type assertion won't fail
if ssquery, ok := squery.(map[string]interface{}); ok {
return ssquery, nil
}
return nil, CANNOT_GET_QUERY_ERROR
}
// MongoDB 3.2+
if squery, ok := query["filter"]; ok {
if ssquery, ok := squery.(map[string]interface{}); ok {
return ssquery, nil
}
return nil, CANNOT_GET_QUERY_ERROR
}
return query, nil
}
// Query is the top level map query element
// Example for MongoDB 3.2+
// "query" : {
// "find" : "col1",
// "filter" : {
// "s2" : {
// "$lt" : "54701",
// "$gte" : "73754"
// }
// },
// "sort" : {
// "user_id" : 1
// }
// }
func fingerprint(query map[string]interface{}) (string, error) {
realQuery, err := getQueryField(query)
if err != nil {
// Try to encode doc.Query as json for prettiness
if buf, err := json.Marshal(realQuery); err == nil {
return "", fmt.Errorf("%v for query %s", err, string(buf))
}
// If we cannot encode as json, return just the error message without the query
return "", err
}
retKeys := keys(realQuery, 0)
sort.Strings(retKeys)
// if there is a sort clause in the query, we have to add all fields in the sort
// fields list that are not in the query keys list (retKeys)
if sortKeys, ok := query["sort"]; ok {
if sortKeysMap, ok := sortKeys.(map[string]interface{}); ok {
sortKeys := mapKeys(sortKeysMap, 0)
for _, sortKey := range sortKeys {
if !inSlice(sortKey, retKeys) {
retKeys = append(retKeys, sortKey)
}
}
}
}
return strings.Join(retKeys, ","), nil
}
func inSlice(str string, list []string) bool {
for _, v := range list {
if v == str {
return true
}
}
return false
}
func keys(query map[string]interface{}, level int) []string {
ks := []string{}
for key, value := range query {
if !shouldIncludeKey(key) {
if shouldSkipKey(key) {
continue
}
ks = append(ks, key)
@@ -595,14 +680,28 @@ func keys(query map[string]interface{}, level int) []string {
return ks
}
func shouldIncludeKey(key string) bool {
filterOut := []string{"shardVersion"}
for _, val := range filterOut {
if val == key {
return false
func mapKeys(query map[string]interface{}, level int) []string {
ks := []string{}
for key, value := range query {
ks = append(ks, key)
if m, ok := value.(map[string]interface{}); ok {
level++
if level <= MAX_DEPTH_LEVEL {
ks = append(ks, keys(m, level)...)
}
}
}
return true
sort.Strings(ks)
return ks
}
func shouldSkipKey(key string) bool {
for _, filter := range keyFilters() {
if matched, _ := regexp.MatchString(filter, key); matched {
return true
}
}
return false
}
func printHeader(opts *options) {

46
src/go/pt-mongodb-query-digest/main_test.go
View File

@@ -188,18 +188,56 @@ func TestFingerprint(t *testing.T) {
},
{
query: map[string]interface{}{"find": "system.profile", "filter": map[string]interface{}{}, "sort": map[string]interface{}{"$natural": 1}},
want: "$natural,filter,find,sort",
want: "$natural",
},
{
query: map[string]interface{}{"collection": "system.profile", "batchSize": 0, "getMore": 18531768265},
want: "batchSize,collection,getMore",
},
/*
Main test case:
Got Query field:
{
"filter": {
"latestFeedbackDate":{
"$gte":1427846400000,
"$lte":1486511999999},
"merchantId":"560bc82a498e0b791959be71",
"reviewed":true,
"serviceFeedback.fiveStarScore.selectedScore":{
"$in":[5,4,3,2,1]
}
},
"find": "saleUpdatedTags",
"ntoreturn":10,
"projection":{
"$sortKey":{
"$meta":"sortKey"
}
},
"shardVersion":[571230652140,"55d1b3f1e6845ce25be7e6db"],
"sort":{"latestFeedbackDate":-1}
}
Want fingerprint:
latestFeedbackDate,merchantId,reviewed,serviceFeedback.fiveStarScore.selectedScore
Why?
1) It is MongoDb 3.2+ (has filter instead of $query)
2) From the "filter" map, we are removing all keys starting with $
3) The key 'latestFeedbackDate' exists in the "sort" map but it is not in the "filter" keys
so it has been added to the final fingerprint
*/
{
query: map[string]interface{}{"sort": map[string]interface{}{"latestFeedbackDate": -1}, "filter": map[string]interface{}{"latestFeedbackDate": map[string]interface{}{"$gte": 1.4278464e+12, "$lte": 1.486511999999e+12}, "merchantId": "560bc82a498e0b791959be71", "reviewed": true, "serviceFeedback.fiveStarScore.selectedScore": map[string]interface{}{"$in": []interface{}{5, 4, 3, 2, 1}}}, "find": "saleUpdatedTags", "ntoreturn": 10, "projection": map[string]interface{}{"$sortKey": map[string]interface{}{"$meta": "sortKey"}}, "shardVersion": []interface{}{5.7123065214e+11, "55d1b3f1e6845ce25be7e6db"}},
want: "latestFeedbackDate,merchantId,reviewed,serviceFeedback.fiveStarScore.selectedScore",
},
}
for _, tt := range tests {
for i, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := fingerprint(tt.query); got != tt.want {
t.Errorf("fingerprint() = %v, want %v", got, tt.want)
if got, err := fingerprint(tt.query); got != tt.want || err != nil {
t.Errorf("fingerprint case #%d:\n got %v,\nwant %v\nerror: %v\n", i, got, tt.want, err)
}
})
}

17
src/go/pt-mongodb-summary/main.go
View File

@@ -27,12 +27,18 @@ import (
const (
TOOLNAME = "pt-mongodb-summary"
DEFAULT_AUTHDB = "admin"
DEFAULT_HOST = "localhost:27017"
DEFAULT_LOGLEVEL = "warn"
DEFAULT_RUNNINGOPSINTERVAL = 1000 // milliseconds
DEFAULT_RUNNINGOPSSAMPLES = 5
)
var (
Version string = "2.2.19"
Build string = "01-01-1980"
GoVersion string = "1.8"
Version string = "3.0.1"
)
type TimedStats struct {
@@ -779,10 +785,11 @@ func externalIP() (string, error) {
func parseFlags() options {
opts := options{
Host: "localhost:27017",
LogLevel: "warn",
RunningOpsSamples: 5,
RunningOpsInterval: 1000, // milliseconds
Host: DEFAULT_HOST,
LogLevel: DEFAULT_LOGLEVEL,
RunningOpsSamples: DEFAULT_RUNNINGOPSSAMPLES,
RunningOpsInterval: DEFAULT_RUNNINGOPSINTERVAL, // milliseconds
AuthDB: DEFAULT_AUTHDB,
}
getopt.BoolVarLong(&opts.Help, "help", 'h', "Show help")

4
src/go/pt-mongodb-summary/templates/clusterwide.go
View File

@@ -10,8 +10,8 @@ Unsharded Collections: {{.UnshardedColsCount}}
Unsharded Data Size: {{.UnshardedDataSizeScaled}} {{.UnshardedDataSizeScale}}
{{- if .Chunks }}
### Chunks:
{{- range .Chunks }}
{{ printf "%30s" .ID}}: {{ printf "%5d" .Count }}
{{ range .Chunks }}
{{- if .ID }} {{ printf "%5d" .Count }} : {{ printf "%-30s" .ID}}{{ end -}}
{{- end -}}
{{ end -}}
`