Added --skip-collections to query-digest

src/go/pt-mongodb-query-digest/README.md

@@ -39,6 +39,7 @@ The last step is sorting the results. The default sort order is by ascending que
 |-n|--limit|show the first n queries|
 |-o|--order-by|comma separated list of order by fields (max values): `count`, `ratio`, `query-time`, `docs-scanned`, `docs-returned`.<br> A `-` in front of the field name denotes reverse order.<br> Example:`--order-by="count,-ratio"`).|
 |-p|--password[=password]|Password (optional). If it is not specified it will be asked|
+|-s|--skip-collections|Comma separated list of collections to skip. Default: `system.profile`|
 |-u|--user|Username|
 |-v|--version|Show version & exit|
 

src/go/pt-mongodb-query-digest/main.go

@@ -45,20 +45,26 @@ type iter interface {
 }
 
 type options struct {
-	AuthDB         string
+	AuthDB          string
-	Database       string
+	Database        string
-	Debug          bool
+	Debug           bool
-	Help           bool
+	Help            bool
-	Host           string
+	Host            string
-	Limit          int
+	Limit           int
-	LogLevel       string
+	LogLevel        string
-	NoVersionCheck bool
+	NoVersionCheck  bool
-	OrderBy        []string
+	OrderBy         []string
-	Password       string
+	Password        string
-	User           string
+	SkipCollections []string
-	Version        bool
+	User            string
+	Version         bool
 }
 
+// This func receives a doc from the profiler and returns:
+// true : the document must be considered
+// false: the document must be skipped
+type docsFilter func(proto.SystemProfile) bool
+
 type statsArray []stat
 
 func (a statsArray) Len() int           { return len(a) }
@@ -129,7 +135,7 @@ func main() {
 
 	opts, err := getOptions()
 	if err != nil {
-		log.Printf("error processing commad line arguments: %s", err)
+		log.Errorf("error processing commad line arguments: %s", err)
 		os.Exit(1)
 	}
 	if opts.Help {
@@ -139,7 +145,7 @@ func main() {
 
 	logLevel, err := log.ParseLevel(opts.LogLevel)
 	if err != nil {
-		fmt.Printf("cannot set log level: %s", err.Error())
+		fmt.Errorf("cannot set log level: %s", err.Error())
 	}
 	log.SetLevel(logLevel)
 
@@ -187,8 +193,36 @@ func main() {
 		os.Exit(5)
 	}
 
-	i := session.DB(di.Database).C("system.profile").Find(bson.M{"op": bson.M{"$nin": []string{"getmore", "delete"}}}).Sort("-$natural").Iter()
+	filters := []docsFilter{}
-	queries := sortQueries(getData(i), opts.OrderBy)
+
+	if len(opts.SkipCollections) > 0 {
+		// Sanitize the param. using --skip-collections="" will produce an 1 element array but
+		// that element will be empty. The same would be using --skip-collections=a,,d
+		cols := []string{}
+		for _, c := range opts.SkipCollections {
+			if strings.TrimSpace(c) != "" {
+				cols = append(cols, c)
+			}
+		}
+		if len(cols) > 0 {
+			// This func receives a doc from the profiler and returns:
+			// true : the document must be considered
+			// false: the document must be skipped
+			filterSystemProfile := func(doc proto.SystemProfile) bool {
+				for _, collection := range cols {
+					if strings.HasSuffix(doc.Ns, collection) {
+						return false
+					}
+				}
+				return true
+			}
+			filters = append(filters, filterSystemProfile)
+		}
+	}
+
+	query := bson.M{"op": bson.M{"$nin": []string{"getmore", "delete"}}}
+	i := session.DB(di.Database).C("system.profile").Find(query).Sort("-$natural").Iter()
+	queries := sortQueries(getData(i, filters), opts.OrderBy)
 
 	uptime := uptime(session)
 
@@ -372,13 +406,24 @@ func calcStats(samples []float64) statistics {
 	return s
 }
 
-func getData(i iter) []stat {
+func getData(i iter, filters []docsFilter) []stat {
 	var doc proto.SystemProfile
 	stats := make(map[groupKey]*stat)
 
 	log.Debug(`Documents returned by db.getSiblinfDB("<dbnamehere>").system.profile.Find({"op": {"$nin": []string{"getmore", "delete"}}).Sort("-$natural")`)
 
 	for i.Next(&doc) && i.Err() == nil {
+		valid := true
+		for _, filter := range filters {
+			if filter(doc) == false {
+				valid = false
+				break
+			}
+		}
+		if !valid {
+			continue
+		}
+
 		log.Debugln("====================================================================================================")
 		log.Debug(pretty.Sprint(doc))
 		if len(doc.Query) > 0 {
@@ -424,7 +469,6 @@ func getData(i iter) []stat {
 
 	// We need to sort the data but a hash cannot be sorted so, convert the hash having
 	// the results to a slice
-
 	sa := statsArray{}
 	for _, s := range stats {
 		sa = append(sa, *s)
@@ -435,7 +479,13 @@ func getData(i iter) []stat {
 }
 
 func getOptions() (*options, error) {
-	opts := &options{Host: "localhost:27017", LogLevel: "warn", OrderBy: []string{"count"}}
+	opts := &options{
+		Host:            "localhost:27017",
+		LogLevel:        "warn",
+		OrderBy:         []string{"count"},
+		SkipCollections: []string{"system.profile"},
+	}
+
 	getopt.BoolVarLong(&opts.Help, "help", '?', "Show help")
 	getopt.BoolVarLong(&opts.Version, "version", 'v', "show version & exit")
 	getopt.BoolVarLong(&opts.NoVersionCheck, "no-version-check", 'c', "Don't check for updates")
@@ -443,6 +493,7 @@ func getOptions() (*options, error) {
 	getopt.IntVarLong(&opts.Limit, "limit", 'n', "show the first n queries")
 
 	getopt.ListVarLong(&opts.OrderBy, "order-by", 'o', "comma separated list of order by fields (max values): count,ratio,query-time,docs-scanned,docs-returned. - in front of the field name denotes reverse order.")
+	getopt.ListVarLong(&opts.SkipCollections, "skip-collections", 's', "comma separated list of collections (namespaces) to skip. Default: system.profile")
 
 	getopt.StringVarLong(&opts.AuthDB, "authenticationDatabase", 'a', "admin", "database used to establish credentials and privileges with a MongoDB server")
 	getopt.StringVarLong(&opts.Database, "database", 'd', "", "database to profile")
@@ -534,7 +585,8 @@ func keys(query map[string]interface{}, level int) []string {
 
 func printHeader(opts *options) {
 	fmt.Printf("%s - %s\n", TOOLNAME, time.Now().Format(time.RFC1123Z))
-	fmt.Printf("Host: %s", opts.Host)
+	fmt.Printf("Host: %s\n", opts.Host)
+	fmt.Printf("Skipping docs in these collections: %v\n", opts.SkipCollections)
 	fmt.Println("")
 }