percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-10 13:11:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes for PT-70, PT-68 & PT-69

Browse Source
This commit is contained in:
Carlos Salguero
2017-02-14 17:46:33 -03:00
parent c027561982
commit 2513815168
5 changed files with 118 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
src/go/pt-mongodb-query-digest/main.go
View File

@@ -189,8 +189,14 @@ func main() {
}
if isProfilerEnabled == false {
log.Error("Profiler is not enabled")
os.Exit(5)
count, err := systemProfileDocsCount(session, di.Database)
if err != nil || count == 0 {
log.Error("Profiler is not enabled")
os.Exit(5)
}
fmt.Printf("Profiler is disabled for the %q database but there are %d documents in the system.profile collection.\n",
di.Database, count)
fmt.Println("Using those documents for the stats")
}
filters := []docsFilter{}
@@ -487,19 +493,22 @@ func getOptions() (*options, error) {
}
getopt.BoolVarLong(&opts.Help, "help", '?', "Show help")
getopt.BoolVarLong(&opts.Version, "version", 'v', "show version & exit")
getopt.BoolVarLong(&opts.NoVersionCheck, "no-version-check", 'c', "Don't check for updates")
getopt.BoolVarLong(&opts.Version, "version", 'v', "Show version & exit")
getopt.BoolVarLong(&opts.NoVersionCheck, "no-version-check", 'c', "Default: Don't check for updates")
getopt.IntVarLong(&opts.Limit, "limit", 'n', "show the first n queries")
getopt.IntVarLong(&opts.Limit, "limit", 'n', "Show the first n queries")
getopt.ListVarLong(&opts.OrderBy, "order-by", 'o', "comma separated list of order by fields (max values): count,ratio,query-time,docs-scanned,docs-returned. - in front of the field name denotes reverse order.")
getopt.ListVarLong(&opts.SkipCollections, "skip-collections", 's', "comma separated list of collections (namespaces) to skip. Default: system.profile")
getopt.ListVarLong(&opts.OrderBy, "order-by", 'o', "Comma separated list of order by fields (max values): "+
"count,ratio,query-time,docs-scanned,docs-returned. "+
"- in front of the field name denotes reverse order.")
getopt.ListVarLong(&opts.SkipCollections, "skip-collections", 's', "A comma separated list of collections (namespaces) to skip."+
" Default: system.profile")
getopt.StringVarLong(&opts.AuthDB, "authenticationDatabase", 'a', "admin", "database used to establish credentials and privileges with a MongoDB server")
getopt.StringVarLong(&opts.Database, "database", 'd', "", "database to profile")
getopt.StringVarLong(&opts.LogLevel, "log-level", 'l', "error", "Log level:, panic, fatal, error, warn, info, debug")
getopt.StringVarLong(&opts.Password, "password", 'p', "", "password").SetOptional()
getopt.StringVarLong(&opts.User, "user", 'u', "username")
getopt.StringVarLong(&opts.AuthDB, "authenticationDatabase", 'a', "admin", "Databaase to use for optional MongoDB authentication. Default: admin")
getopt.StringVarLong(&opts.Database, "database", 'd', "", "MongoDB database to profile")
getopt.StringVarLong(&opts.LogLevel, "log-level", 'l', "Log level: error", "panic, fatal, error, warn, info, debug. Default: error")
getopt.StringVarLong(&opts.Password, "password", 'p', "", "Password to use for optional MongoDB authentication").SetOptional()
getopt.StringVarLong(&opts.User, "username", 'u', "Username to use for optional MongoDB authentication")
getopt.SetParameters("host[:port]/database")
@@ -547,7 +556,7 @@ func getDialInfo(opts *options) *mgo.DialInfo {
di, _ := mgo.ParseURL(opts.Host)
di.FailFast = true
if getopt.IsSet("user") {
if getopt.IsSet("username") {
di.Username = opts.User
}
if getopt.IsSet("password") {
@@ -571,6 +580,9 @@ func fingerprint(query map[string]interface{}) string {
func keys(query map[string]interface{}, level int) []string {
ks := []string{}
for key, value := range query {
if !shouldIncludeKey(key) {
continue
}
ks = append(ks, key)
if m, ok := value.(map[string]interface{}); ok {
level++
@@ -583,10 +595,20 @@ func keys(query map[string]interface{}, level int) []string {
return ks
}
func shouldIncludeKey(key string) bool {
filterOut := []string{"shardVersion"}
for _, val := range filterOut {
if val == key {
return false
}
}
return true
}
func printHeader(opts *options) {
fmt.Printf("%s - %s\n", TOOLNAME, time.Now().Format(time.RFC1123Z))
fmt.Printf("Host: %s\n", opts.Host)
fmt.Printf("Skipping docs in these collections: %v\n", opts.SkipCollections)
fmt.Printf("Skipping profiled queries on these collections: %v\n", opts.SkipCollections)
fmt.Println("")
}
@@ -778,22 +800,44 @@ func isProfilerEnabled(dialer pmgo.Dialer, di *mgo.DialInfo) (bool, error) {
if err != nil {
return false, err
}
for _, member := range replicaMembers {
if member.State == proto.REPLICA_SET_MEMBER_PRIMARY {
di.Addrs = []string{member.Name}
session, err := dialer.DialWithInfo(di)
if err != nil {
continue
}
defer session.Close()
if err := session.DB(di.Database).Run(bson.M{"profile": -1}, &ps); err != nil {
continue
}
if ps.Was == 0 {
return false, nil
}
for _, member := range replicaMembers {
// Stand alone instances return state = REPLICA_SET_MEMBER_STARTUP
di.Addrs = []string{member.Name}
session, err := dialer.DialWithInfo(di)
if err != nil {
continue
}
defer session.Close()
session.SetMode(mgo.Monotonic, true)
isReplicaEnabled := isReplicasetEnabled(session)
if member.StateStr == "configsvr" {
continue
}
if isReplicaEnabled && member.State != proto.REPLICA_SET_MEMBER_PRIMARY {
continue
}
if err := session.DB(di.Database).Run(bson.M{"profile": -1}, &ps); err != nil {
continue
}
if ps.Was == 0 {
return false, nil
}
}
return true, nil
}
func systemProfileDocsCount(session pmgo.SessionManager, dbname string) (int, error) {
return session.DB(dbname).C("system.profile").Count()
}
func isReplicasetEnabled(session pmgo.SessionManager) bool {
rss := proto.ReplicaSetStatus{}
if err := session.Run(bson.M{"replSetGetStatus": 1}, &rss); err != nil {
return false
}
return true
}

51
src/go/pt-mongodb-query-digest/main_test.go
View File

@@ -10,7 +10,9 @@ import (
"time"
"github.com/percona/percona-toolkit/src/go/mongolib/proto"
"github.com/percona/pmgo"
mgo "gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/dbtest"
)
@@ -50,7 +52,7 @@ func TestMain(m *testing.M) {
func TestCalcStats(t *testing.T) {
it := Server.Session().DB("samples").C("system_profile").Find(nil).Sort("Ts").Iter()
data := getData(it)
data := getData(it, []docsFilter{})
s := calcStats(data[0].NScanned)
want := statistics{Pct: 0, Total: 159, Min: 79, Max: 80, Avg: 79.5, Pct95: 80, StdDev: 0.5, Median: 79.5}
@@ -116,27 +118,25 @@ func TestGetData(t *testing.T) {
i: it,
want: []stat{
stat{
ID: "6c3fff4804febd156700a06f9a346162",
Fingerprint: "find,limit",
Namespace: "samples.col1",
Query: map[string]interface{}{
"find": "col1",
"limit": float64(2),
},
ID: "6c3fff4804febd156700a06f9a346162",
Operation: "query",
Fingerprint: "find,limit",
Namespace: "samples.col1",
Query: map[string]interface{}{"find": "col1", "limit": float64(2)},
Count: 2,
TableScan: false,
NScanned: []float64{79, 80},
NReturned: []float64{79, 80},
QueryTime: []float64{27, 28},
ResponseLength: []float64{109, 110},
LockTime: nil,
BlockedTime: nil,
LockTime: times(nil),
BlockedTime: times(nil),
FirstSeen: time.Date(2016, time.November, 8, 13, 46, 27, 0, time.UTC).Local(),
LastSeen: time.Date(2016, time.November, 8, 13, 46, 27, 0, time.UTC).Local(),
},
stat{
ID: "fdcea004122ddb225bc56de417391e25",
Operation: "query",
Fingerprint: "find",
Namespace: "samples.col1",
Query: map[string]interface{}{"find": "col1"},
@@ -146,8 +146,8 @@ func TestGetData(t *testing.T) {
NReturned: []float64{71, 72, 73, 74, 75, 76, 77, 78},
QueryTime: []float64{19, 20, 21, 22, 23, 24, 25, 26},
ResponseLength: []float64{101, 102, 103, 104, 105, 106, 107, 108},
LockTime: nil,
BlockedTime: nil,
LockTime: times(nil),
BlockedTime: times(nil),
FirstSeen: time.Date(2016, time.November, 8, 13, 46, 27, 0, time.UTC).Local(),
LastSeen: time.Date(2016, time.November, 8, 13, 46, 27, 0, time.UTC).Local(),
},
@@ -157,17 +157,17 @@ func TestGetData(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := getData(tt.i)
got := getData(tt.i, []docsFilter{})
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("got\n%#v\nwant\n%#v", got, tt.want)
}
})
}
}
func TestUptime(t *testing.T) {
session := Server.Session()
session := pmgo.NewSessionManager(Server.Session())
time.Sleep(1500 * time.Millisecond)
if uptime(session) <= 0 {
t.Error("uptime is 0")
@@ -288,3 +288,22 @@ func TestTimesLess(t *testing.T) {
})
}
}
func TestIsProfilerEnabled(t *testing.T) {
mongoDSN := os.Getenv("PT_TEST_MONGODB_DSN")
if mongoDSN == "" {
t.Skip("Skippping TestIsProfilerEnabled. It runs only in integration tests")
}
dialer := pmgo.NewDialer()
di, _ := mgo.ParseURL(mongoDSN)
enabled, err := isProfilerEnabled(dialer, di)
if err != nil {
t.Errorf("Cannot check if profiler is enabled: %s", err.Error())
}
if enabled != true {
t.Error("Profiler must be enabled")
}
}

6
src/go/pt-mongodb-query-digest/vendor/vendor.json vendored
View File

@@ -63,10 +63,10 @@
"revisionTime": "2017-02-01T15:06:01Z"
},
{
"checksumSHA1": "NviDzeLQRdyp9Ux2vMvu4zQvyYo=",
"checksumSHA1": "rDaYK9qS/RbE7w8jiPktdA3+lc0=",
"path": "github.com/percona/pmgo",
"revision": "2650f7f1545746eddae964e7308440900684c21a",
"revisionTime": "2017-02-10T14:26:46Z"
"revision": "9eabc5fda168f8d9961a56a767d8304e1b65135b",
"revisionTime": "2017-02-14T09:40:05Z"
},
{
"checksumSHA1": "ynJSWoF6v+3zMnh9R0QmmG6iGV8=",

10
src/go/pt-mongodb-summary/main.go
View File

@@ -787,13 +787,13 @@ func parseFlags() options {
getopt.BoolVarLong(&opts.Help, "help", 'h', "Show help")
getopt.BoolVarLong(&opts.Version, "version", 'v', "", "Show version & exit")
getopt.BoolVarLong(&opts.NoVersionCheck, "no-version-check", 'c', "", "Don't check for updates")
getopt.BoolVarLong(&opts.NoVersionCheck, "no-version-check", 'c', "", "Default: Don't check for updates")
getopt.StringVarLong(&opts.User, "user", 'u', "", "User name")
getopt.StringVarLong(&opts.Password, "password", 'p', "", "Password").SetOptional()
getopt.StringVarLong(&opts.User, "username", 'u', "", "Username to use for optional MongoDB authentication")
getopt.StringVarLong(&opts.Password, "password", 'p', "", "Password to use for optional MongoDB authentication").SetOptional()
getopt.StringVarLong(&opts.AuthDB, "authenticationDatabase", 'a', "admin",
"Database used to establish credentials and privileges with a MongoDB server")
getopt.StringVarLong(&opts.LogLevel, "log-level", 'l', "error", "Log level:, panic, fatal, error, warn, info, debug")
"Databaase to use for optional MongoDB authentication. Default: admin")
getopt.StringVarLong(&opts.LogLevel, "log-level", 'l', "error", "Log level: panic, fatal, error, warn, info, debug. Default: error")
getopt.IntVarLong(&opts.RunningOpsSamples, "running-ops-samples", 's',
fmt.Sprintf("Number of samples to collect for running ops. Default: %d", opts.RunningOpsSamples))

6
src/go/pt-mongodb-summary/vendor/vendor.json vendored
View File

@@ -63,10 +63,10 @@
"revisionTime": "2017-02-01T15:06:01Z"
},
{
"checksumSHA1": "NviDzeLQRdyp9Ux2vMvu4zQvyYo=",
"checksumSHA1": "rDaYK9qS/RbE7w8jiPktdA3+lc0=",
"path": "github.com/percona/pmgo",
"revision": "2650f7f1545746eddae964e7308440900684c21a",
"revisionTime": "2017-02-10T14:26:46Z"
"revision": "9eabc5fda168f8d9961a56a767d8304e1b65135b",
"revisionTime": "2017-02-14T09:40:05Z"
},
{
"checksumSHA1": "kTY8RZr5j26RNIR+F2SnMLeB7G8=",