Merge branch '3.0' into pt-3.0.5-pre-release

docs/pt-mongodb-query-digest.rst

@@ -45,6 +45,10 @@ Options
 ``-d``, ``--database``
   Specifies which database to profile
 
+``-f``, ``--output-format``
+  Specifies the report output format. Valid options are: ``text``, ``json``.
+  The default value is ``text``.
+
 ``-l``, ``--log-level``
   Specifies the log level:
   ``panic``, ``fatal``, ``error``, ``warn``, ``info``, ``debug error``

docs/pt-mongodb-summary.rst

@@ -35,6 +35,10 @@ Options
   with a MongoDB server.
   By default, the ``admin`` database is used.
 
+``-f``, ``--output-format``
+  Specifies the report output format. Valid options are: ``text``, ``json``.
+  The default value is ``text``.
+
 ``-p``, ``--password``
   Specifies the password to use when connecting to a server
   with authentication enabled.

src/go/pt-mongodb-query-digest/README.md

@@ -38,6 +38,7 @@ The last step is sorting the results. The default sort order is by ascending que
 |-a|--authenticationDatabase|database used to establish credentials and privileges with a MongoDB server admin|
 |-c|--no-version-check|Don't check for updates|
 |-d|--database|database to profile|
+|-f|--output-format|report output format. Valid values are text, json. Default: text|
 |-l|--log-level|Log level:, panic, fatal, error, warn, info, debug error|
 |-n|--limit|show the first n queries|
 |-o|--order-by|comma separated list of order by fields (max values): `count`, `ratio`, `query-time`, `docs-scanned`, `docs-returned`.<br> A `-` in front of the field name denotes reverse order.<br> Example:`--order-by="count,-ratio"`).|

src/go/pt-mongodb-query-digest/main.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
 	"os"
 	"sort"
@@ -50,6 +52,7 @@ type options struct {
 	LogLevel        string
 	NoVersionCheck  bool
 	OrderBy         []string
+	OutputFormat    string
 	Password        string
 	SkipCollections []string
 	SSLCAFile       string
@@ -58,6 +61,12 @@ type options struct {
 	Version         bool
 }
 
+type report struct {
+	Headers     []string
+	QueryStats  []stats.QueryStats
+	QueryTotals stats.QueryStats
+}
+
 func main() {
 
 	opts, err := getOptions()
@@ -149,26 +158,58 @@ func main() {
 	queriesStats := queries.CalcQueriesStats(uptime)
 	sortedQueryStats := sortQueries(queriesStats, opts.OrderBy)
 
-	printHeader(opts)
-
-	queryTotals := queries.CalcTotalQueriesStats(uptime)
-	tt, _ := template.New("query").Funcs(template.FuncMap{
-		"Format": format,
-	}).Parse(getTotalsTemplate())
-	tt.Execute(os.Stdout, queryTotals)
-
-	t, _ := template.New("query").Funcs(template.FuncMap{
-		"Format": format,
-	}).Parse(getQueryTemplate())
-
 	if opts.Limit > 0 && len(sortedQueryStats) > opts.Limit {
 		sortedQueryStats = sortedQueryStats[:opts.Limit]
 	}
-	for i, qs := range sortedQueryStats {
-		qs.Rank = i + 1
-		t.Execute(os.Stdout, qs)
+
+	if len(queries) == 0 {
+		log.Errorf("No queries found in profiler information for database %q\n", di.Database)
+		return
+	}
+	rep := report{
+		Headers:     getHeaders(opts),
+		QueryTotals: queries.CalcTotalQueriesStats(uptime),
+		QueryStats:  sortedQueryStats,
 	}
 
+	out, err := formatResults(rep, opts.OutputFormat)
+	if err != nil {
+		log.Errorf("Cannot parse the report: %s", err.Error())
+		os.Exit(5)
+	}
+
+	fmt.Println(string(out))
+
+}
+
+func formatResults(rep report, outputFormat string) ([]byte, error) {
+	var buf *bytes.Buffer
+
+	switch outputFormat {
+	case "json":
+		b, err := json.MarshalIndent(rep, "", "    ")
+		if err != nil {
+			return nil, fmt.Errorf("[Error] Cannot convert results to json: %s", err.Error())
+		}
+		buf = bytes.NewBuffer(b)
+	default:
+		buf = new(bytes.Buffer)
+
+		tt, _ := template.New("query").Funcs(template.FuncMap{
+			"Format": format,
+		}).Parse(getTotalsTemplate())
+		tt.Execute(buf, rep.QueryTotals)
+
+		t, _ := template.New("query").Funcs(template.FuncMap{
+			"Format": format,
+		}).Parse(getQueryTemplate())
+
+		for _, qs := range rep.QueryStats {
+			t.Execute(buf, qs)
+		}
+	}
+
+	return buf.Bytes(), nil
 }
 
 // format scales a number and returns a string made of the scaled value and unit (K=Kilo, M=Mega, T=Tera)
@@ -233,6 +274,7 @@ func getOptions() (*options, error) {
 	gop.StringVarLong(&opts.AuthDB, "authenticationDatabase", 'a', "admin", "Database to use for optional MongoDB authentication. Default: admin")
 	gop.StringVarLong(&opts.Database, "database", 'd', "", "MongoDB database to profile")
 	gop.StringVarLong(&opts.LogLevel, "log-level", 'l', "Log level: error", "panic, fatal, error, warn, info, debug. Default: error")
+	gop.StringVarLong(&opts.OutputFormat, "output-format", 'f', "text", "Output format: text, json. Default: text")
 	gop.StringVarLong(&opts.Password, "password", 'p', "", "Password to use for optional MongoDB authentication").SetOptional()
 	gop.StringVarLong(&opts.User, "username", 'u', "Username to use for optional MongoDB authentication")
 	gop.StringVarLong(&opts.SSLCAFile, "sslCAFile", 0, "SSL CA cert file used for authentication")
@@ -265,6 +307,11 @@ func getOptions() (*options, error) {
 		}
 	}
 
+	if opts.OutputFormat != "json" && opts.OutputFormat != "text" {
+		opts.OutputFormat = "text"
+		log.Infof("Invalid output format '%s'. Using text format", opts.OutputFormat)
+	}
+
 	if gop.IsSet("password") && opts.Password == "" {
 		print("Password: ")
 		pass, err := gopass.GetPasswd()
@@ -307,11 +354,13 @@ func getDialInfo(opts *options) *pmgo.DialInfo {
 	return pmgoDialInfo
 }
 
-func printHeader(opts *options) {
-	fmt.Printf("%s - %s\n", TOOLNAME, time.Now().Format(time.RFC1123Z))
-	fmt.Printf("Host: %s\n", opts.Host)
-	fmt.Printf("Skipping profiled queries on these collections: %v\n", opts.SkipCollections)
-	fmt.Println("")
+func getHeaders(opts *options) []string {
+	h := []string{
+		fmt.Sprintf("%s - %s\n", TOOLNAME, time.Now().Format(time.RFC1123Z)),
+		fmt.Sprintf("Host: %s\n", opts.Host),
+		fmt.Sprintf("Skipping profiled queries on these collections: %v\n", opts.SkipCollections),
+	}
+	return h
 }
 
 func getQueryTemplate() string {
@@ -482,6 +531,7 @@ func isProfilerEnabled(dialer pmgo.Dialer, di *pmgo.DialInfo) (bool, error) {
 	for _, member := range replicaMembers {
 		// Stand alone instances return state = REPLICA_SET_MEMBER_STARTUP
 		di.Addrs = []string{member.Name}
+		di.Direct = true
 		session, err := dialer.DialWithInfo(di)
 		if err != nil {
 			continue

src/go/pt-mongodb-query-digest/main_test.go

@@ -96,6 +96,7 @@ func TestParseArgs(t *testing.T) {
 				OrderBy:         strings.Split(DEFAULT_ORDERBY, ","),
 				SkipCollections: strings.Split(DEFAULT_SKIPCOLLECTIONS, ","),
 				AuthDB:          DEFAULT_AUTHDB,
+				OutputFormat:    "text",
 			},
 		},
 		{
@@ -111,6 +112,7 @@ func TestParseArgs(t *testing.T) {
 				SkipCollections: strings.Split(DEFAULT_SKIPCOLLECTIONS, ","),
 				AuthDB:          DEFAULT_AUTHDB,
 				Help:            false,
+				OutputFormat:    "text",
 			},
 		},
 	}

src/go/pt-mongodb-summary/README.rst

@@ -15,15 +15,14 @@ Please check the `releases <https://github.com/percona/toolkit-go/releases>`_ ta
 
 Paramters
 ^^^^^^^^^
-===== ========= ======= ================================================================================
-Short Long      Default Description
-===== ========= ======= ================================================================================ 
-u     user      empty   user name to use when connecting if DB auth is enabled
-p     password  empty   password to use when connecting if DB auth is enabled
-a     auth-db   admin   database used to establish credentials and privileges with a MongoDB server
-===== ========= ======= ================================================================================
+|Short|Long|Default|Description|
+|-----|----|-------|-----------|
+|-a|--auth-db|admin|database used to establish credentials and privileges with a MongoDB server|
+|-f|--output-format|report output format|Valid values are text, json. Default: text|
+|-f|--output-format|text|output format: text, json. Default: text|
+|-p|--password|empty|password to use when connecting if DB auth is enabled|
+|-u|--user|empty|user name to use when connecting if DB auth is enabled|
 
-| 
 
 ``-p`` is an optional parameter. If it is used it shouldn't have a blank between the parameter and its value: `-p<password>`  
 It can be also used as `-p` without specifying a password; in that case, the program will ask the password to avoid using a password in the command line.