Merge branch '3.0' into PT-141

Changelog

@@ -1,5 +1,9 @@
 Changelog for Percona Toolkit
 
+v3.0.4
+
+ * Fixed bug PT-138 : Added --output-format option to pt-mongodb-summary
+
 v3.0.3
 
  * Fixed bug PT-133 : Sandbox won't start correctly if autocommit=0 in my.cnf

bin/pt-archiver

@@ -45,7 +45,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.2';
+our $VERSION = '3.0.3';
 
 use strict;
 use warnings FATAL => 'all';

bin/pt-config-diff

@@ -43,7 +43,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.2';
+our $VERSION = '3.0.3';
 
 use strict;
 use warnings FATAL => 'all';

bin/pt-deadlock-logger

@@ -42,7 +42,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.2';
+our $VERSION = '3.0.3';
 
 use strict;
 use warnings FATAL => 'all';

bin/pt-diskstats

@@ -38,7 +38,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.2';
+our $VERSION = '3.0.3';
 
 use strict;
 use warnings FATAL => 'all';

bin/pt-duplicate-key-checker

@@ -39,7 +39,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.2';
+our $VERSION = '3.0.3';
 
 use strict;
 use warnings FATAL => 'all';

bin/pt-find

@@ -35,7 +35,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.2';
+our $VERSION = '3.0.3';
 
 use strict;
 use warnings FATAL => 'all';

bin/pt-fk-error-logger

@@ -37,7 +37,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.2';
+our $VERSION = '3.0.3';
 
 use strict;
 use warnings FATAL => 'all';

bin/pt-heartbeat

@@ -44,7 +44,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.2';
+our $VERSION = '3.0.3';
 
 use strict;
 use warnings FATAL => 'all';

bin/pt-index-usage

@@ -45,7 +45,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.2';
+our $VERSION = '3.0.3';
 
 use strict;
 use warnings FATAL => 'all';

bin/pt-kill

@@ -47,7 +47,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.2';
+our $VERSION = '3.0.3';
 
 use strict;
 use warnings FATAL => 'all';

bin/pt-online-schema-change

@@ -55,7 +55,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.2';
+our $VERSION = '3.0.3';
 
 use strict;
 use warnings FATAL => 'all';
@@ -10053,8 +10053,13 @@ sub find_renamed_cols {
    /x;
 
    my $table_ident  = qr/$unquoted_ident|`$quoted_ident`|"$ansi_quotes_ident"/;
-   my $alter_change_col_re = qr/\bCHANGE \s+ (?:COLUMN \s+)? (?:COMMENT\s+[^\]['"].*?[^\]['"])?
+
-                                    ($table_ident) \s+ ($table_ident)/ix;
+   # remove comments
+   $alter =~ s/^(.*?)\s+COMMENT\s+'(.*?[^\\]')+(.*)/$1$3/; 
+   $alter =~ s/^(.*?)\s+COMMENT\s+"(.*?[^\\]")+(.*)/$1$3/; 
+
+   my $alter_change_col_re = qr/\bCHANGE \s+ (?:COLUMN \s+)? 
+                                ($table_ident) \s+ ($table_ident)/ix;
 
    my %renames;
    while ( $alter =~ /$alter_change_col_re/g ) {

bin/pt-query-digest

@@ -64,7 +64,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.2';
+our $VERSION = '3.0.3';
 
 use strict;
 use warnings FATAL => 'all';

bin/pt-slave-delay

@@ -40,7 +40,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.2';
+our $VERSION = '3.0.3';
 
 use strict;
 use warnings FATAL => 'all';

bin/pt-slave-restart

@@ -41,7 +41,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.2';
+our $VERSION = '3.0.3';
 
 use strict;
 use warnings FATAL => 'all';

bin/pt-table-checksum

@@ -57,7 +57,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.2';
+our $VERSION = '3.0.3';
 
 use strict;
 use warnings FATAL => 'all';

bin/pt-table-sync

@@ -55,7 +55,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.2';
+our $VERSION = '3.0.3';
 
 use strict;
 use warnings FATAL => 'all';

bin/pt-upgrade

@@ -61,7 +61,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.2';
+our $VERSION = '3.0.3';
 
 use strict;
 use warnings FATAL => 'all';

bin/pt-variable-advisor

@@ -44,7 +44,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.2';
+our $VERSION = '3.0.3';
 
 use strict;
 use warnings FATAL => 'all';

lib/Percona/Toolkit.pm

@@ -18,7 +18,7 @@
 # ###########################################################################
 package Percona::Toolkit;
 
-our $VERSION = '3.0.2';
+our $VERSION = '3.0.3';
 
 use strict;
 use warnings FATAL => 'all';

src/go/pt-mongodb-summary/main.go

@@ -1,11 +1,13 @@
 package main
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
+	"html/template"
 	"net"
 	"os"
 	"strings"
-	"text/template"
 	"time"
 
 	version "github.com/hashicorp/go-version"
@@ -33,6 +35,7 @@ const (
 	DEFAULT_LOGLEVEL           = "warn"
 	DEFAULT_RUNNINGOPSINTERVAL = 1000 // milliseconds
 	DEFAULT_RUNNINGOPSSAMPLES  = 5
+	DEFAULT_OUTPUT_FORMAT      = "text"
 )
 
 var (
@@ -130,12 +133,24 @@ type options struct {
 	Version            bool
 	NoVersionCheck     bool
 	NoRunningOps       bool
+	OutputFormat       string
 	RunningOpsSamples  int
 	RunningOpsInterval int
 	SSLCAFile          string
 	SSLPEMKeyFile      string
 }
 
+type collectedInfo struct {
+	BalancerStats    *proto.BalancerStats
+	ClusterWideInfo  *clusterwideInfo
+	OplogInfo        []proto.OplogInfo
+	ReplicaMembers   []proto.Members
+	RunningOps       *opCounters
+	SecuritySettings *security
+	HostInfo         *hostInfo
+	Errors           []string
+}
+
 func main() {
 
 	opts, err := parseFlags()
@@ -210,77 +225,105 @@ func main() {
 	defer session.Close()
 	session.SetMode(mgo.Monotonic, true)
 
-	hostInfo, err := GetHostinfo(session)
+	ci := &collectedInfo{}
+
+	ci.HostInfo, err = GetHostinfo(session)
 	if err != nil {
 		message := fmt.Sprintf("Cannot get host info for %q: %s", di.Addrs[0], err.Error())
 		log.Errorf(message)
 		os.Exit(2)
 	}
 
-	if replicaMembers, err := util.GetReplicasetMembers(dialer, di); err != nil {
+	if ci.ReplicaMembers, err = util.GetReplicasetMembers(dialer, di); err != nil {
 		log.Warnf("[Error] cannot get replicaset members: %v\n", err)
 		os.Exit(2)
-	} else {
-		log.Debugf("replicaMembers:\n%+v\n", replicaMembers)
-		t := template.Must(template.New("replicas").Parse(templates.Replicas))
-		t.Execute(os.Stdout, replicaMembers)
 	}
-
+	log.Debugf("replicaMembers:\n%+v\n", ci.ReplicaMembers)
-	// Host Info
-	t := template.Must(template.New("hosttemplateData").Parse(templates.HostInfo))
-	t.Execute(os.Stdout, hostInfo)
 
 	if opts.RunningOpsSamples > 0 && opts.RunningOpsInterval > 0 {
-		if rops, err := GetOpCountersStats(session, opts.RunningOpsSamples, time.Duration(opts.RunningOpsInterval)*time.Millisecond); err != nil {
+		if ci.RunningOps, err = GetOpCountersStats(session, opts.RunningOpsSamples, time.Duration(opts.RunningOpsInterval)*time.Millisecond); err != nil {
 			log.Printf("[Error] cannot get Opcounters stats: %v\n", err)
-		} else {
-			t := template.Must(template.New("runningOps").Parse(templates.RunningOps))
-			t.Execute(os.Stdout, rops)
 		}
 	}
 
-	if hostInfo != nil {
+	if ci.HostInfo != nil {
-		if security, err := GetSecuritySettings(session, hostInfo.Version); err != nil {
+		if ci.SecuritySettings, err = GetSecuritySettings(session, ci.HostInfo.Version); err != nil {
 			log.Errorf("[Error] cannot get security settings: %v\n", err)
-		} else {
-			t := template.Must(template.New("ssl").Parse(templates.Security))
-			t.Execute(os.Stdout, security)
 		}
 	} else {
 		log.Warn("Cannot check security settings since host info is not available (permissions?)")
 	}
 
-	if oplogInfo, err := oplog.GetOplogInfo(hostnames, di); err != nil {
+	if ci.OplogInfo, err = oplog.GetOplogInfo(hostnames, di); err != nil {
 		log.Info("Cannot get Oplog info: %v\n", err)
 	} else {
-		if len(oplogInfo) > 0 {
+		if len(ci.OplogInfo) == 0 {
-			t := template.Must(template.New("oplogInfo").Parse(templates.Oplog))
-			t.Execute(os.Stdout, oplogInfo[0])
-		} else {
-
 			log.Info("oplog info is empty. Skipping")
+		} else {
+			ci.OplogInfo = ci.OplogInfo[:1]
 		}
 	}
 
 	// individual servers won't know about this info
-	if hostInfo.NodeType == "mongos" {
+	if ci.HostInfo.NodeType == "mongos" {
-		if cwi, err := GetClusterwideInfo(session); err != nil {
+		if ci.ClusterWideInfo, err = GetClusterwideInfo(session); err != nil {
 			log.Printf("[Error] cannot get cluster wide info: %v\n", err)
-		} else {
-			t := template.Must(template.New("clusterwide").Parse(templates.Clusterwide))
-			t.Execute(os.Stdout, cwi)
 		}
 	}
 
-	if hostInfo.NodeType == "mongos" {
+	if ci.HostInfo.NodeType == "mongos" {
-		if bs, err := GetBalancerStats(session); err != nil {
+		if ci.BalancerStats, err = GetBalancerStats(session); err != nil {
 			log.Printf("[Error] cannot get balancer stats: %v\n", err)
-		} else {
-			t := template.Must(template.New("balancer").Parse(templates.BalancerStats))
-			t.Execute(os.Stdout, bs)
 		}
 	}
 
+	out, err := formatResults(ci, opts.OutputFormat)
+	if err != nil {
+		log.Errorf("Cannot format the results: %s", err.Error())
+		os.Exit(1)
+	}
+	fmt.Println(string(out))
+
+}
+
+func formatResults(ci *collectedInfo, format string) ([]byte, error) {
+	var buf *bytes.Buffer
+
+	switch format {
+	case "json":
+		b, err := json.MarshalIndent(ci, "", "    ")
+		if err != nil {
+			return nil, fmt.Errorf("[Error] Cannot convert results to json: %s", err.Error())
+		}
+		buf = bytes.NewBuffer(b)
+	default:
+		buf = new(bytes.Buffer)
+
+		t := template.Must(template.New("replicas").Parse(templates.Replicas))
+		t.Execute(buf, ci.ReplicaMembers)
+
+		t = template.Must(template.New("hosttemplateData").Parse(templates.HostInfo))
+		t.Execute(buf, ci.HostInfo)
+
+		t = template.Must(template.New("runningOps").Parse(templates.RunningOps))
+		t.Execute(buf, ci.RunningOps)
+
+		t = template.Must(template.New("ssl").Parse(templates.Security))
+		t.Execute(buf, ci.SecuritySettings)
+
+		if ci.OplogInfo != nil && len(ci.OplogInfo) > 0 {
+			t = template.Must(template.New("oplogInfo").Parse(templates.Oplog))
+			t.Execute(buf, ci.OplogInfo[0])
+		}
+
+		t = template.Must(template.New("clusterwide").Parse(templates.Clusterwide))
+		t.Execute(buf, ci.ClusterWideInfo)
+
+		t = template.Must(template.New("balancer").Parse(templates.BalancerStats))
+		t.Execute(buf, ci.BalancerStats)
+	}
+
+	return buf.Bytes(), nil
 }
 
 func GetHostinfo(session pmgo.SessionManager) (*hostInfo, error) {
@@ -472,6 +515,7 @@ func GetSecuritySettings(session pmgo.SessionManager, ver string) (*security, er
 	// Lets try both
 	newSession := session.Clone()
 	defer newSession.Close()
+
 	newSession.SetMode(mgo.Strong, true)
 
 	if s.Users, s.Roles, err = getUserRolesCount(newSession); err != nil {
@@ -811,6 +855,7 @@ func parseFlags() (*options, error) {
 		RunningOpsSamples:  DEFAULT_RUNNINGOPSSAMPLES,
 		RunningOpsInterval: DEFAULT_RUNNINGOPSINTERVAL, // milliseconds
 		AuthDB:             DEFAULT_AUTHDB,
+		OutputFormat:       DEFAULT_OUTPUT_FORMAT,
 	}
 
 	gop := getopt.New()
@@ -821,8 +866,9 @@ func parseFlags() (*options, error) {
 	gop.StringVarLong(&opts.User, "username", 'u', "", "Username to use for optional MongoDB authentication")
 	gop.StringVarLong(&opts.Password, "password", 'p', "", "Password to use for optional MongoDB authentication").SetOptional()
 	gop.StringVarLong(&opts.AuthDB, "authenticationDatabase", 'a', "admin",
-		"Databaae to use for optional MongoDB authentication. Default: admin")
+		"Database to use for optional MongoDB authentication. Default: admin")
 	gop.StringVarLong(&opts.LogLevel, "log-level", 'l', "error", "Log level: panic, fatal, error, warn, info, debug. Default: error")
+	gop.StringVarLong(&opts.OutputFormat, "output-format", 'f', "text", "Output format: text, json. Default: text")
 
 	gop.IntVarLong(&opts.RunningOpsSamples, "running-ops-samples", 's',
 		fmt.Sprintf("Number of samples to collect for running ops. Default: %d", opts.RunningOpsSamples))
@@ -852,6 +898,9 @@ func parseFlags() (*options, error) {
 		gop.PrintUsage(os.Stdout)
 		return nil, nil
 	}
+	if opts.OutputFormat != "json" && opts.OutputFormat != "text" {
+		log.Infof("Invalid output format '%s'. Using text format", opts.OutputFormat)
+	}
 
 	return opts, nil
 }

src/go/pt-mongodb-summary/main_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 	"time"
 
+	mgo "gopkg.in/mgo.v2"
 	"gopkg.in/mgo.v2/bson"
 	"gopkg.in/mgo.v2/dbtest"
 
@@ -205,6 +206,9 @@ func TestSecurityOpts(t *testing.T) {
 		session.EXPECT().DB("admin").Return(database)
 		database.EXPECT().Run(bson.D{{"getCmdLineOpts", 1}, {"recordStats", 1}}, gomock.Any()).SetArg(1, cmd)
 
+		session.EXPECT().Clone().Return(session)
+		session.EXPECT().SetMode(mgo.Strong, true)
+
 		session.EXPECT().DB("admin").Return(database)
 		database.EXPECT().C("system.users").Return(usersCol)
 		usersCol.EXPECT().Count().Return(1, nil)
@@ -212,6 +216,7 @@ func TestSecurityOpts(t *testing.T) {
 		session.EXPECT().DB("admin").Return(database)
 		database.EXPECT().C("system.roles").Return(rolesCol)
 		rolesCol.EXPECT().Count().Return(2, nil)
+		session.EXPECT().Close().Return()
 
 		got, err := GetSecuritySettings(session, "3.2")
 
@@ -392,21 +397,25 @@ func TestParseArgs(t *testing.T) {
 		{
 			args: []string{TOOLNAME}, // arg[0] is the command itself
 			want: &options{
-				Host:     DEFAULT_HOST,
+				Host:               DEFAULT_HOST,
-				LogLevel: DEFAULT_LOGLEVEL,
+				LogLevel:           DEFAULT_LOGLEVEL,
-				AuthDB:   DEFAULT_AUTHDB,
+				AuthDB:             DEFAULT_AUTHDB,
+				RunningOpsSamples:  DEFAULT_RUNNINGOPSSAMPLES,
+				RunningOpsInterval: DEFAULT_RUNNINGOPSINTERVAL,
+				OutputFormat:       "text",
 			},
 		},
 		{
 			args: []string{TOOLNAME, "zapp.brannigan.net:27018/samples", "--help"},
-			want: &options{
+			want: nil,
-				Host:     "zapp.brannigan.net:27018/samples",
-				LogLevel: DEFAULT_LOGLEVEL,
-				AuthDB:   DEFAULT_AUTHDB,
-				Help:     true,
-			},
 		},
 	}
+
+	// Capture stdout to not to show help
+	old := os.Stdout // keep backup of the real stdout
+	_, w, _ := os.Pipe()
+	os.Stdout = w
+
 	for i, test := range tests {
 		getopt.Reset()
 		os.Args = test.args
@@ -419,4 +428,6 @@ func TestParseArgs(t *testing.T) {
 		}
 	}
 
+	os.Stdout = old
+
 }

t/pt-online-schema-change/bugs.t

@@ -610,7 +610,7 @@ $sb->load_file('master', "$sample/bug-1613915.sql");
 $output = output(
    sub { pt_online_schema_change::main(@args, "$master_dsn,D=test,t=o1",
          '--execute', 
-         '--alter', "ADD COLUMN c INT COMMENT 'change plus more than one word'",
+         '--alter', "ADD COLUMN c INT COMMENT 'change \"plus\" more than one word'",
          '--chunk-size', '10', '--no-check-alter',
          ),
       },
@@ -633,7 +633,7 @@ is(
 $rows = $master_dbh->selectrow_arrayref("SHOW CREATE TABLE test.o1");
 like(
       $rows->[1],
-      qr/COMMENT 'change plus more than one word'/,
+      qr/COMMENT 'change "plus" more than one word'/,
       "recognize comments",
 );