Fixed merge conflicts and changelog

Co-authored-by: rasika-chivate <95711051+rasika-chivate@users.noreply.github.com>

.gitignore

@@ -12,6 +12,7 @@ Makefile.old
 bin/pt-mongodb-*
 bin/pt-secure-*
 bin/pt-pg-*
+bin/pt-k8s-debug-collector
 !src/go/pt-mongodb-query-digest/vendor/vendor.json
 !src/go/pt-mongodb-summary/vendor/vendor.json
 src/go/pt-mongodb-query-digest/vendor/

Changelog

@@ -1,21 +1,28 @@
 Changelog for Percona Toolkit
 
-v3.3.2 WIP release date not set yet
+v3.4.0
 
+ * Fixed bug   PT-1218: pt-stalk ominous open_tables function
+ * Fixed bug   PT-1336: pt-stalk removes user's files from the destination directory
+ * Fixed bug   PT-1398: pt-stalk gets the incorrect mysqld pid when the host installed a multi MySQL instance
  * Improvement PT-1417: Inconsistent creation of toolkit tables
  * Fixed bug   PT-1627:	pt-mysql-summary doesn't verify which version of jemalloc is in use
  * Fixed bug   PT-1747: pt-online-schema-change: metadata lock can break database for rebuild_constraints
- * Improvement PT-1800: PTDEBUG=1 exposes passwords
+ * Improvement PT-1800: The environment variable PTDEBUG=1 exposes the passwords
- * Fixed bug   PT-1900:	pt-query-digest not hiding parameter properly sometimes when parameter=binary
+ * Improvement PT-1887: pt-diskstat is not working for new kernels
- * Improvement PT-1940:	ptsoc dropswap with mysql8: revise rejection (Thanks duxthefux)
+ * Fixed bug   PT-1900: At times, pt-query-digest does not hide the parameters properly when parameter=binary
+ * Improvement PT-1940:	ptsoc dropswap method that was rejected for Mysql 8 has been fixed with Mysql 8.0.14
  * Fixed bug   PT-1953: pt-summary typo: Memory mamagement.
  * Fixed bug   PT-1959: go part of the toolkit still has the version 3.3.0
- * Fixed bug   PT-1965: pt-stalk --mysql-only doesn't collect mysqladmin outputs (Thanks Sergey Kuzmichev)
+ * Fixed bug   PT-1965: pt-stalk --mysql-only doesn't collect mysqladmin output (Thanks Sergey Kuzmichev)
  * Fixed bug   PT-1966: Test no_drop_no_swap for the pt-online-schema-change is broken (Thanks Tsubasa Tanaka)
  * Fixed bug   PT-1974: Support fingerprinting for --print in pt-kill
+ * New Tool    PT-1978: Add reporting on unused/redundant indexes for MongoDB by pt-mongodb-summary
+ * Improvement PT-1979: Add gathering of admin parameters for MongoDB by pt-mongodb-summary
  * Fixed bug   PT-1983:	pt-summary missing one DIMM	
- * Admin task  PT-2009: Update Go Lang dependencies
+ * Fixed bug   PT-2016: pt-table-checksum fails to build replace query when table lacks primary key
- * Admin task  PT-2011:	Make build reproducible
+ * Fixed bug   PT-2023: pt-upgrade Error: Wide character in print
+ * Improvement PT-2037: Add option --skip-mysql or --system-only for pt-stalk
 
 v3.3.1 release 2021-04-28 
 

Makefile.PL

@@ -2,7 +2,7 @@ use ExtUtils::MakeMaker;
 
 WriteMakefile(
     NAME      => 'percona-toolkit',
-    VERSION   => '3.3.1',
+    VERSION   => '3.4.0',
     EXE_FILES => [ <bin/*> ],
     MAN1PODS  => {
       'docs/percona-toolkit.pod' => 'blib/man1/percona-toolkit.1p',

bin/pt-align

@@ -1364,6 +1364,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-align 3.3.2
+pt-align 3.4.0
 
 =cut

bin/pt-archiver

@@ -8661,6 +8661,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-archiver 3.3.2
+pt-archiver 3.4.0
 
 =cut

bin/pt-config-diff

@@ -5917,6 +5917,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-config-diff 3.3.2
+pt-config-diff 3.4.0
 
 =cut

bin/pt-deadlock-logger

@@ -5710,6 +5710,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-deadlock-logger 3.3.2
+pt-deadlock-logger 3.4.0
 
 =cut

bin/pt-diskstats

@@ -5684,6 +5684,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-diskstats 3.3.2
+pt-diskstats 3.4.0
 
 =cut

bin/pt-duplicate-key-checker

@@ -5771,6 +5771,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-duplicate-key-checker 3.3.2
+pt-duplicate-key-checker 3.4.0
 
 =cut

bin/pt-fifo-split

@@ -1653,6 +1653,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-fifo-split 3.3.2
+pt-fifo-split 3.4.0
 
 =cut

bin/pt-find

@@ -5132,6 +5132,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-find 3.3.2
+pt-find 3.4.0
 
 =cut

bin/pt-fingerprint

@@ -2262,6 +2262,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-fingerprint 3.3.2
+pt-fingerprint 3.4.0
 
 =cut

bin/pt-fk-error-logger

@@ -4693,6 +4693,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-fk-error-logger 3.3.2
+pt-fk-error-logger 3.4.0
 
 =cut

bin/pt-heartbeat

@@ -7394,6 +7394,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-heartbeat 3.3.2
+pt-heartbeat 3.4.0
 
 =cut

bin/pt-index-usage

@@ -7705,6 +7705,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-index-usage 3.3.2
+pt-index-usage 3.4.0
 
 =cut

bin/pt-ioprofile

@@ -1132,7 +1132,7 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-ioprofile 3.3.2
+pt-ioprofile 3.4.0
 
 =cut
 

bin/pt-kill

@@ -8576,6 +8576,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-kill 3.3.2
+pt-kill 3.4.0
 
 =cut

bin/pt-mext

@@ -809,7 +809,7 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-mext 3.3.2
+pt-mext 3.4.0
 
 =cut
 

bin/pt-mysql-summary

@@ -3296,7 +3296,7 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-mysql-summary 3.3.2
+pt-mysql-summary 3.4.0
 
 =cut
 

bin/pt-online-schema-change

@@ -13469,6 +13469,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-online-schema-change 3.3.2
+pt-online-schema-change 3.4.0
 
 =cut

bin/pt-pmp

@@ -901,7 +901,7 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-pmp 3.3.2
+pt-pmp 3.4.0
 
 =cut
 

bin/pt-query-digest

@@ -16977,6 +16977,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-query-digest 3.3.2
+pt-query-digest 3.4.0
 
 =cut

bin/pt-show-grants

@@ -2618,6 +2618,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-show-grants 3.3.2
+pt-show-grants 3.4.0
 
 =cut

bin/pt-sift

@@ -1250,7 +1250,7 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-sift 3.3.2
+pt-sift 3.4.0
 
 =cut
 

bin/pt-slave-delay

@@ -4993,6 +4993,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-slave-delay 3.3.2
+pt-slave-delay 3.4.0
 
 =cut

bin/pt-slave-find

@@ -4528,6 +4528,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-slave-find 3.3.2
+pt-slave-find 3.4.0
 
 =cut

bin/pt-slave-restart

@@ -6164,6 +6164,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-slave-restart 3.3.2
+pt-slave-restart 3.4.0
 
 =cut

bin/pt-stalk

@@ -2486,7 +2486,7 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-stalk 3.3.2
+pt-stalk 3.4.0
 
 =cut
 

bin/pt-summary

@@ -2729,7 +2729,7 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-summary 3.3.2
+pt-summary 3.4.0
 
 =cut
 

bin/pt-table-checksum

@@ -14188,6 +14188,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-table-checksum 3.3.2
+pt-table-checksum 3.4.0
 
 =cut

bin/pt-table-sync

@@ -13101,6 +13101,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-table-sync 3.3.2
+pt-table-sync 3.4.0
 
 =cut

bin/pt-table-usage

@@ -8519,6 +8519,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-table-usage 3.3.2
+pt-table-usage 3.4.0
 
 =cut

bin/pt-upgrade

@@ -11454,6 +11454,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-upgrade 3.3.2
+pt-upgrade 3.4.0
 
 =cut

bin/pt-variable-advisor

@@ -6262,6 +6262,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-variable-advisor 3.3.2
+pt-variable-advisor 3.4.0
 
 =cut

bin/pt-visual-explain

@@ -3308,6 +3308,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-visual-explain 3.3.2
+pt-visual-explain 3.4.0
 
 =cut

config/sphinx-build/conf.py

@@ -48,9 +48,9 @@ copyright = u'2021, Percona LLC and/or its affiliates'
 # built documents.
 #
 # The short X.Y version.
-version = '3.3'
+version = '3.4'
 # The full version, including alpha/beta/rc tags.
-release = '3.3.1'
+release = '3.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/percona-toolkit.pod

@@ -567,6 +567,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-Percona Toolkit v3.3.1 released 2021-04-22
+Percona Toolkit v3.4.0 released 2022-04-05
 
 =cut

go.mod

@@ -5,6 +5,7 @@ go 1.17
 require (
 	github.com/Masterminds/semver v1.4.2
 	github.com/alecthomas/kingpin v2.2.6+incompatible
+	github.com/alecthomas/kong v0.5.0
 	github.com/go-ini/ini v1.66.4
 	github.com/golang/mock v1.4.4
 	github.com/google/uuid v1.3.0
@@ -40,9 +41,11 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.15.1 // indirect
 	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rogpeppe/go-internal v1.6.1 // indirect
 	github.com/smartystreets/goconvey v1.7.2 // indirect
 	github.com/tklauser/go-sysconf v0.3.10 // indirect
 	github.com/tklauser/numcpus v0.4.0 // indirect

go.sum

@@ -13,6 +13,9 @@ github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDO
 github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
 github.com/alecthomas/kingpin v2.2.6+incompatible h1:5svnBTFgJjZvGKyYBtMB0+m5wvrbUHiqye8wRJMlnYI=
 github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE=
+github.com/alecthomas/kong v0.5.0 h1:u8Kdw+eeml93qtMZ04iei0CFYve/WPcA5IFh+9wSskE=
+github.com/alecthomas/kong v0.5.0/go.mod h1:uzxf/HUh0tj43x1AyJROl3JT7SgsZ5m+icOv1csRhc0=
+github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRYS3XieqF+Z9B9gNxo/eANAJCF2eiN4=
@@ -242,6 +245,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/shirou/gopsutil v2.20.8+incompatible h1:8c7Atn0FAUZJo+f4wYbN0iVpdWniCQk7IYwGtgdh1mY=
 github.com/shirou/gopsutil v2.20.8+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=

lib/Percona/Toolkit.pm

@@ -18,7 +18,7 @@
 # ###########################################################################
 package Percona::Toolkit;
 
-our $VERSION = '3.3.2';
+our $VERSION = '3.4.0';
 
 use strict;
 use warnings FATAL => 'all';

src/go/mongolib/proto/collstats.go

@@ -13,8 +13,7 @@ type ShardStas struct {
 	UserFlags         int64    `json:"userFlags"`
 	Capped            bool     `json:"capped"`
 	Nindexes          int64    `json:"nindexes"`
-	IndexDetails      struct {
+	IndexDetails      struct{} `json:"indexDetails"`
-	} `json:"indexDetails"`
 	TotalIndexSize    int64    `json:"totalIndexSize"`
 	IndexSizes        struct {
 		ID       int64 `json:"_id_"`

src/go/pt-k8s-debug-collector/dumper/dumper.go

@@ -72,7 +72,7 @@ func New(location, namespace, resource string) Dumper {
 		cmd:       "kubectl",
 		resources: resources,
 		location:  "cluster-dump",
-		mode:      int64(0777),
+		mode:      int64(0o777),
 		namespace: namespace,
 		crType:    resource,
 	}

src/go/pt-mongodb-index-check/README.md

@@ -0,0 +1,48 @@
+# pt-mongodb-index-check
+
+
+
+## Introduction
+
+This tool can perform checks on MongoDB indexes. 
+
+Currently, these checks are available:
+
+### Duplicated indexes
+
+Check for indexes that are the prefix of other indexes. For example if we have these 2 indexes
+
+```
+db.getSiblingDB("testdb").test_col.createIndex({"f1": 1, "f2": -1, "f3": 1, "f4": 1}, {"name": "idx_01"});
+db.getSiblingDB("testdb").test_col.createIndex({"f1": 1, "f2": -1, "f3": 1}, {"name": "idx_02"});
+```
+
+The index `idx_02` is the prefix of `idx_01` because it has the same keys in the same order so, `idx_02` can be dropped.
+
+### Unused indexes.
+
+This check gets the `$indexstats` for all indexes and reports those having `accesses.ops` = 0.
+
+## Usage
+
+Run the program as `pt-mongodb-index-check <command> [flags]`
+
+#### Available commands
+
+| Command          | Description                        |
+| ---------------- | ---------------------------------- |
+| check-duplicated | Run checks for duplicated indexes. |
+| check-unused     | Run check for unused indexes.      |
+| check-all        | Run all checks                     |
+
+#### Available flags
+
+| Flag | Description |
+| ---- | ----------- |
+|--all-databases|Check in all databases excluding system dbs.|
+|--databases=DATABASES,...|Comma separated list of databases to check.|
+|--all-collections|Check in all collections in the selected databases.|
+|--collections=COLLECTIONS,...|Comma separated list of collections to check.|
+|--mongodb.uri=<connection string>|Connection URI|
+|--json|Show output as JSON|
+

src/go/pt-mongodb-index-check/indexes/duplicated.go

@@ -0,0 +1,103 @@
+package indexes
+
+import (
+	"context"
+	"log"
+	"sort"
+	"strings"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+type collectionIndex struct {
+	Name      string      `bson:"name"`
+	Namespace string      `bson:"ns"`
+	V         int         `bson:"v"`
+	Key       primitive.D `bson:"key"`
+}
+
+func (di collectionIndex) ComparableKey() string {
+	str := ""
+	for _, elem := range di.Key {
+		str += sign(elem) + elem.Key
+	}
+	return str
+}
+
+func sign(elem primitive.E) string {
+	sign := "+"
+	switch elem.Value.(type) {
+	case int32: // internal MongoDB indexes like _id_ or lastUsed have the sign field as int32.
+		if elem.Value.(int32) < 0 {
+			sign = "-"
+		}
+	case float64: // All other indexes have the sign field as float64.
+		if elem.Value.(float64) < 0 {
+			sign = "-"
+		}
+	}
+	return sign
+}
+
+// IndexKey holds the list of fields that are part of an index, along with the field order.
+type IndexKey []primitive.E
+
+// String returns the index fields as a string. The + sign means ascending on this field
+// and a - sign indicates a descending order for that field.
+func (di IndexKey) String() string {
+	str := ""
+	for _, elem := range di {
+		str += sign(elem) + elem.Key + " "
+	}
+
+	return str
+}
+
+// DuplicateIndex represents a duplicated index pair.
+// An index is considered as the duplicate of another one if it is it's prefix.
+// Example: the index +f1-f2 is the prefix of +f1-f2+f3.
+type Duplicate struct {
+	Namespace     string
+	Name          string
+	Key           IndexKey
+	ContainerName string
+	ContainerKey  IndexKey
+}
+
+func FindDuplicated(ctx context.Context, client *mongo.Client, database, collection string) ([]Duplicate, error) {
+	di := []Duplicate{}
+
+	cursor, err := client.Database(database).Collection(collection).Indexes().List(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	var results []collectionIndex
+	if err = cursor.All(context.TODO(), &results); err != nil {
+		log.Fatal(err)
+	}
+
+	sort.Slice(results, func(i, j int) bool {
+		return results[i].ComparableKey() < results[j].ComparableKey()
+	})
+
+	for i := 0; i < len(results)-1; i++ {
+		for j := i + 1; j < len(results); j++ {
+			if strings.HasPrefix(results[j].ComparableKey(), results[i].ComparableKey()) {
+				idx := Duplicate{
+					Namespace:     database + "." + collection,
+					Name:          results[i].Name,
+					Key:           make([]primitive.E, len(results[i].Key)),
+					ContainerName: results[j].Name,
+					ContainerKey:  make([]primitive.E, len(results[j].Key)),
+				}
+				copy(idx.Key, results[i].Key)
+				copy(idx.ContainerKey, results[j].Key)
+				di = append(di, idx)
+			}
+		}
+	}
+
+	return di, nil
+}

src/go/pt-mongodb-index-check/indexes/duplicated_test.go

@@ -0,0 +1,117 @@
+package indexes
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/AlekSi/pointer"
+	"github.com/stretchr/testify/assert"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"gopkg.in/mgo.v2/bson"
+
+	tu "github.com/percona/percona-toolkit/src/go/internal/testutils"
+)
+
+func TestDuplicateIndexes(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	client, err := tu.TestClient(ctx, tu.MongoDBShard1PrimaryPort)
+	if err != nil {
+		t.Fatalf("cannot get a new MongoDB client: %s", err)
+	}
+
+	dbname := "test_db"
+	collname := "test_col"
+
+	database := client.Database(dbname)
+	database.Drop(ctx)       //nolint:errcheck
+	defer database.Drop(ctx) //nolint:errcheck
+
+	_, err = database.Collection(collname).InsertOne(ctx, bson.M{"f1": 1, "f2": "2", "f3": "a", "f4": "c"})
+	assert.NoError(t, err)
+
+	testCases := []primitive.D{
+		{{"f1", 1}, {"f2", -1}, {"f3", 1}, {"f4", 1}},
+		{{"f1", 1}, {"f2", -1}, {"f3", 1}, {"f4", 1}}, // this will throw a duplicate index error
+		{{"f1", 1}, {"f2", -1}, {"f3", 1}},
+		{{"f1", 1}, {"f2", -1}},
+		{{"f3", -1}},
+	}
+
+	errCount := 0
+	for i, tc := range testCases {
+		mod := mongo.IndexModel{
+			Keys: tc,
+			Options: &options.IndexOptions{
+				Name: pointer.ToString(fmt.Sprintf("idx_%02d", i)),
+			},
+		}
+		_, err := database.Collection(collname).Indexes().CreateOne(ctx, mod)
+		if err != nil {
+			errCount++
+		}
+	}
+	/*
+	 At this point we have 5 indexes: _id: (MongoDB's default), idx_00, idx_02, idx_03, idx_04.
+	 idx_01 wasn't created since it duplicates idx_00 and errCount=1.
+	*/
+
+	assert.Equal(t, 1, errCount)
+
+	want := []Duplicate{
+		{
+			Name:      "idx_03",
+			Namespace: "test_db.test_col",
+			Key: IndexKey{
+				{Key: "f1", Value: int32(1)},
+				{Key: "f2", Value: int32(-1)},
+			},
+			ContainerName: "idx_02",
+			ContainerKey: IndexKey{
+				{Key: "f1", Value: int32(1)},
+				{Key: "f2", Value: int32(-1)},
+				{Key: "f3", Value: int32(1)},
+			},
+		},
+		{
+			Name:      "idx_03",
+			Namespace: "test_db.test_col",
+			Key: IndexKey{
+				{Key: "f1", Value: int32(1)},
+				{Key: "f2", Value: int32(-1)},
+			},
+			ContainerName: "idx_00",
+			ContainerKey: IndexKey{
+				{Key: "f1", Value: int32(1)},
+				{Key: "f2", Value: int32(-1)},
+				{Key: "f3", Value: int32(1)},
+				{Key: "f4", Value: int32(1)},
+			},
+		},
+		{
+			Name:      "idx_02",
+			Namespace: "test_db.test_col",
+			Key: IndexKey{
+				{Key: "f1", Value: int32(1)},
+				{Key: "f2", Value: int32(-1)},
+				{Key: "f3", Value: int32(1)},
+			},
+			ContainerName: "idx_00",
+			ContainerKey: IndexKey{
+				{Key: "f1", Value: int32(1)},
+				{Key: "f2", Value: int32(-1)},
+				{Key: "f3", Value: int32(1)},
+				{Key: "f4", Value: int32(1)},
+			},
+		},
+	}
+
+	di, err := FindDuplicated(ctx, client, dbname, collname)
+	assert.NoError(t, err)
+	assert.Equal(t, want, di)
+}

src/go/pt-mongodb-index-check/indexes/unused.go

@@ -0,0 +1,62 @@
+package indexes
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+	"gopkg.in/mgo.v2/bson"
+)
+
+var systemDBs = []string{"admin", "config", "local", "system.profile"} //nolint:gochecknoglobals
+// IndexStat hold an index usage statistics.
+type IndexStat struct {
+	Accesses struct {
+		Ops   int64              `bson:"ops"`
+		Since primitive.DateTime `bson:"since"`
+	} `bson:"accesses"`
+	Spec struct {
+		Name      string      `bson:"name"`
+		Namespace string      `bson:"ns"`
+		V         int32       `bson:"v"`
+		Key       primitive.D `bson:"key"`
+	} `bson:"spec"`
+	Name string      `bson:"name"`
+	Key  primitive.D `bson:"key"`
+	Host string      `bson:"host"`
+}
+
+func in(search string, items []string) bool {
+	for _, item := range items {
+		if search == item {
+			return true
+		}
+	}
+	return false
+}
+
+// FindUnusedIndexes returns a list of unused indexes for the given database and collection.
+func FindUnused(ctx context.Context, client *mongo.Client, database, collection string) ([]IndexStat, error) {
+	aggregation := mongo.Pipeline{
+		{{Key: "$indexStats", Value: primitive.M{}}},
+		{{Key: "$match", Value: primitive.M{"accesses.ops": 0}}},
+		{{Key: "$match", Value: primitive.M{"name": bson.M{"$ne": "_id_"}}}},
+	}
+
+	if in(database, systemDBs) {
+		return nil, nil
+	}
+
+	cursor, err := client.Database(database).Collection(collection).Aggregate(ctx, aggregation)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot run $indexStats for unused indexes")
+	}
+
+	var stats []IndexStat
+	if err = cursor.All(ctx, &stats); err != nil {
+		return nil, errors.Wrap(err, "cannot get $indexStats for unused indexes")
+	}
+
+	return stats, nil
+}

src/go/pt-mongodb-index-check/indexes/unused_test.go

@@ -0,0 +1,82 @@
+package indexes
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/AlekSi/pointer"
+	"github.com/stretchr/testify/assert"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"gopkg.in/mgo.v2/bson"
+
+	tu "github.com/percona/percona-toolkit/src/go/internal/testutils"
+)
+
+func TestUnusedIndexes(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	client, err := tu.TestClient(ctx, tu.MongoDBShard1PrimaryPort)
+	if err != nil {
+		t.Fatalf("cannot get a new MongoDB client: %s", err)
+	}
+
+	dbname := "test_db"
+	collname := "test_col"
+
+	database := client.Database(dbname)
+	database.Drop(ctx)       //nolint:errcheck
+	defer database.Drop(ctx) //nolint:errcheck
+
+	testCases := []primitive.D{
+		{{"f1", 1}, {"f2", -1}, {"f3", 1}, {"f4", 1}},
+		{{"f3", -1}},
+		{{"f4", -1}},
+	}
+
+	errCount := 0
+	for i, tc := range testCases {
+		mod := mongo.IndexModel{
+			Keys: tc,
+			Options: &options.IndexOptions{
+				Name: pointer.ToString(fmt.Sprintf("idx_%02d", i)),
+			},
+		}
+		_, err := database.Collection(collname).Indexes().CreateOne(ctx, mod)
+		if err != nil {
+			errCount++
+		}
+	}
+
+	for i := 0; i < 100; i++ {
+		_, err = database.Collection(collname).InsertOne(ctx,
+			bson.M{"f1": rand.Int63n(1000), "f2": rand.Int63n(1000), "f3": rand.Int63n(1000), "f4": rand.Int63n(1000)})
+		assert.NoError(t, err)
+	}
+
+	// Make use of idx_02: {"f4": -1} to exclude it from the results so we ensure only unused indexes
+	// are being listed.
+	_, err = database.Collection(collname).Find(ctx, primitive.M{"f4": primitive.M{"$gt": 500}})
+	assert.NoError(t, err)
+
+	want := []string{"idx_00", "idx_01"}
+
+	ui, err := FindUnused(ctx, client, dbname, collname)
+	assert.NoError(t, err)
+
+	got := make([]string, 0, len(ui))
+	for _, idx := range ui {
+		// compare only names because the index struct has a timestamp in it and it is variable.
+		got = append(got, idx.Name)
+	}
+
+	sort.Strings(got)
+
+	assert.Equal(t, want, got)
+}

src/go/pt-mongodb-index-check/main.go

@@ -0,0 +1,188 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"text/template"
+	"time"
+
+	"github.com/alecthomas/kong"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+
+	"github.com/percona/percona-toolkit/src/go/pt-mongodb-index-check/indexes"
+	"github.com/percona/percona-toolkit/src/go/pt-mongodb-index-check/templates"
+)
+
+type cmdlineArgs struct {
+	CheckUnused     struct{} `cmd:"" name:"check-unused" help:"Check for unused indexes."`
+	CheckDuplicated struct{} `cmd:"" name:"check-duplicates" help:"Check for duplicated indexes."`
+	CheckAll        struct{} `cmd:"" name:"check-all" help:"Check for unused and duplicated indexes."`
+	ShowHelp        struct{} `cmd:"" default:"1"`
+	Version         struct{} `cmd:"" name:"version"`
+
+	AllDatabases bool     `name:"all-databases" xor:"db" help:"Check in all databases excluding system dbs"`
+	Databases    []string `name:"databases" xor:"db" help:"Comma separated list of databases to check"`
+
+	AllCollections bool     `name:"all-collections" xor:"colls" help:"Check in all collections in the selected databases."`
+	Collections    []string `name:"collections" xor:"colls" help:"Comma separated list of collections to check"`
+	URI            string   `name:"mongodb.uri" required:"" placeholder:"mongodb://host:port/admindb?options" help:"Connection URI"`
+	JSON           bool     `name:"json" help:"Show output as JSON"`
+}
+
+type response struct {
+	Unused     []indexes.IndexStat
+	Duplicated []indexes.Duplicate
+}
+
+const (
+	TOOLNAME = "pt-mongodb-index-check"
+)
+
+var (
+	Build     string = "2020-04-23" //nolint
+	GoVersion string = "1.14.1"     //nolint
+	Version   string = "3.4.0"      //nolint
+	Commit    string                //nolint
+)
+
+func main() {
+	var args cmdlineArgs
+	kongctx := kong.Parse(&args, kong.UsageOnError())
+
+	if kongctx.Command() == "version" {
+		fmt.Println(TOOLNAME)
+		fmt.Printf("Version %s\n", Version)
+		fmt.Printf("Build: %s using %s\n", Build, GoVersion)
+		fmt.Printf("Commit: %s\n", Commit)
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+
+	if !strings.HasPrefix(args.URI, "mongodb") && !strings.HasPrefix(args.URI, "mongodb+srv") {
+		args.URI = "mongodb://" + args.URI
+	}
+
+	client, err := mongo.Connect(ctx, options.Client().ApplyURI(args.URI))
+	if err != nil {
+		log.Fatalf("Cannot connect to the database: %q", err)
+	}
+
+	if args.AllDatabases {
+		args.Databases, err = client.ListDatabaseNames(context.TODO(), primitive.D{})
+		if err != nil {
+			log.Fatalf("cannot list all databases: %s", err)
+		}
+	}
+	if args.AllCollections {
+		args.Collections = nil
+	}
+
+	resp := response{}
+
+	switch kongctx.Command() {
+	case "check-unused":
+		resp.Unused = findUnused(ctx, client, args.Databases, args.Collections)
+	case "check-duplicates":
+		resp.Duplicated = findDuplicated(ctx, client, args.Databases, args.Collections)
+	case "check-all":
+		resp.Unused = findUnused(ctx, client, args.Databases, args.Collections)
+		resp.Duplicated = findDuplicated(ctx, client, args.Databases, args.Collections)
+	default:
+		kong.DefaultHelpPrinter(kong.HelpOptions{}, kongctx)
+	}
+
+	fmt.Println(output(resp, args.JSON))
+}
+
+func output(resp response, asJson bool) string {
+	if asJson {
+		jsonStr, err := json.MarshalIndent(resp, "", "\t")
+		if err != nil {
+			log.Fatal("cannot encode the response as json")
+		}
+		return string(jsonStr)
+	}
+
+	buf := new(bytes.Buffer)
+
+	t := template.Must(template.New("duplicated").Parse(templates.Duplicated))
+	if err := t.Execute(buf, resp.Duplicated); err != nil {
+		log.Fatal(errors.Wrap(err, "cannot parse clusterwide section of the output template"))
+	}
+
+	t = template.Must(template.New("unused").Parse(templates.Unused))
+	if err := t.Execute(buf, resp.Unused); err != nil {
+		log.Fatal(errors.Wrap(err, "cannot parse clusterwide section of the output template"))
+	}
+
+	return buf.String()
+}
+
+func findUnused(ctx context.Context, client *mongo.Client, databases []string, collections []string) []indexes.IndexStat {
+	unused := []indexes.IndexStat{}
+	var err error
+
+	colls := make([]string, len(collections))
+	copy(colls, collections)
+
+	for _, database := range databases {
+		if len(collections) == 0 {
+			colls, err = client.Database(database).ListCollectionNames(ctx, primitive.D{})
+			if err != nil {
+				log.Errorf("cannot get the list of collections for the database %s", database)
+				continue
+			}
+		}
+
+		for _, collection := range colls {
+			idx, err := indexes.FindUnused(ctx, client, database, collection)
+			if err != nil {
+				log.Errorf("error while checking unused indexes in %s.%s: %s", database, collection, err)
+				continue
+			}
+
+			unused = append(unused, idx...)
+		}
+	}
+
+	return unused
+}
+
+func findDuplicated(ctx context.Context, client *mongo.Client, databases []string, collections []string) []indexes.Duplicate {
+	duplicated := []indexes.Duplicate{}
+	var err error
+
+	colls := make([]string, len(collections))
+	copy(colls, collections)
+
+	for _, database := range databases {
+		if len(collections) == 0 {
+			colls, err = client.Database(database).ListCollectionNames(ctx, primitive.D{})
+			if err != nil {
+				log.Errorf("cannot get the list of collections for the database %s", database)
+				continue
+			}
+		}
+
+		for _, collection := range colls {
+			dups, err := indexes.FindDuplicated(ctx, client, database, collection)
+			if err != nil {
+				log.Errorf("error while checking duplicated indexes in %s.%s: %s", database, collection, err)
+				continue
+			}
+
+			duplicated = append(duplicated, dups...)
+		}
+	}
+
+	return duplicated
+}

src/go/pt-mongodb-index-check/templates/duplicated.go

@@ -0,0 +1,10 @@
+package templates
+
+// {{if $i}},{{end}} adds a comma after the first element.
+// When $i == 0 (first element) {{ if $i }} returns false (0)
+
+var Duplicated = `
+Duplicated indexes 
+{{ range . }}
+{{ .Namespace }}, index '{{ .Name }}', with fields { {{- range $i, $val := .Key }}{{if $i}}, {{end}}{{ $val.Key }}:{{ $val.Value }}{{ end -}} } is the prefix of '{{ .ContainerName }}' with fields { {{- range $i, $val := .ContainerKey }}{{if $i}}, {{end}}{{ $val.Key }}:{{ $val.Value }}{{ end -}} }{{ end}}
+`

src/go/pt-mongodb-index-check/templates/unused.go

@@ -0,0 +1,10 @@
+package templates
+
+// {{if $i}},{{end}} adds a comma after the first element.
+// When $i == 0 (first element) {{ if $i }} returns false (0)
+
+var Unused = `
+Unused indexes since last restart
+{{ range . }}
+{{ .Spec.Namespace }}, index '{{ .Name }}' with fields { {{- range $i, $val := .Key }}{{if $i}}, {{end}}{{ $val.Key }}:{{ $val.Value }} }{{ end }}{{ end}}
+`

src/go/pt-mongodb-query-digest/main.go

@@ -13,6 +13,11 @@ import (
 
 	"github.com/howeyc/gopass"
 	"github.com/pborman/getopt"
+	log "github.com/sirupsen/logrus"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+
 	"github.com/percona/percona-toolkit/src/go/lib/config"
 	"github.com/percona/percona-toolkit/src/go/lib/versioncheck"
 	"github.com/percona/percona-toolkit/src/go/mongolib/fingerprinter"
@@ -21,10 +26,6 @@ import (
 	"github.com/percona/percona-toolkit/src/go/mongolib/stats"
 	"github.com/percona/percona-toolkit/src/go/mongolib/util"
 	"github.com/percona/percona-toolkit/src/go/pt-mongodb-query-digest/filter"
-	log "github.com/sirupsen/logrus"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	"go.mongodb.org/mongo-driver/mongo"
-	"go.mongodb.org/mongo-driver/mongo/options"
 )
 
 const (
@@ -40,7 +41,7 @@ const (
 var (
 	Build     string = "2020-04-23" //nolint
 	GoVersion string = "1.14.1"     //nolint
-	Version   string = "3.3.2"      //nolint
+	Version   string = "3.4.0"      //nolint
 	Commit    string                //nolint
 )
 

src/go/pt-mongodb-summary/main.go

@@ -19,18 +19,19 @@ import (
 	version "github.com/hashicorp/go-version"
 	"github.com/howeyc/gopass"
 	"github.com/pborman/getopt"
-	"github.com/percona/percona-toolkit/src/go/lib/config"
-	"github.com/percona/percona-toolkit/src/go/lib/versioncheck"
-	"github.com/percona/percona-toolkit/src/go/mongolib/proto"
-	"github.com/percona/percona-toolkit/src/go/mongolib/util"
-	"github.com/percona/percona-toolkit/src/go/pt-mongodb-summary/oplog"
-	"github.com/percona/percona-toolkit/src/go/pt-mongodb-summary/templates"
 	"github.com/pkg/errors"
 	"github.com/shirou/gopsutil/process"
 	log "github.com/sirupsen/logrus"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
+
+	"github.com/percona/percona-toolkit/src/go/lib/config"
+	"github.com/percona/percona-toolkit/src/go/lib/versioncheck"
+	"github.com/percona/percona-toolkit/src/go/mongolib/proto"
+	"github.com/percona/percona-toolkit/src/go/mongolib/util"
+	"github.com/percona/percona-toolkit/src/go/pt-mongodb-summary/oplog"
+	"github.com/percona/percona-toolkit/src/go/pt-mongodb-summary/templates"
 )
 
 const (
@@ -56,7 +57,7 @@ const (
 var (
 	Build     string = "2020-04-23"
 	GoVersion string = "1.14.1"
-	Version   string = "3.3.2"
+	Version   string = "3.4.0"
 	Commit    string
 
 	defaultConnectionTimeout = 3 * time.Second
@@ -615,7 +616,8 @@ func getNodeType(ctx context.Context, client *mongo.Client) (string, error) {
 }
 
 func getOpCountersStats(ctx context.Context, client *mongo.Client, count int,
-	sleep time.Duration) (*opCounters, error) {
+	sleep time.Duration,
+) (*opCounters, error) {
 	oc := &opCounters{}
 	prevOpCount := &opCounters{}
 	ss := proto.ServerStatus{}

src/go/pt-mongodb-summary/main_test.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/pborman/getopt"
+
 	tu "github.com/percona/percona-toolkit/src/go/internal/testutils"
 	"github.com/percona/percona-toolkit/src/go/mongolib/proto"
 )

src/go/pt-mongodb-summary/oplog/oplog.go

@@ -6,12 +6,13 @@ import (
 	"sort"
 	"time"
 
-	"github.com/percona/percona-toolkit/src/go/mongolib/proto"
-	"github.com/percona/percona-toolkit/src/go/mongolib/util"
 	"github.com/pkg/errors"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
+
+	"github.com/percona/percona-toolkit/src/go/mongolib/proto"
+	"github.com/percona/percona-toolkit/src/go/mongolib/util"
 )
 
 func GetOplogInfo(ctx context.Context, hostnames []string, co *options.ClientOptions) ([]proto.OplogInfo, error) {

src/go/pt-pg-summary/main.go

@@ -8,20 +8,20 @@ import (
 	"text/template"
 
 	"github.com/alecthomas/kingpin"
+	_ "github.com/lib/pq"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	"github.com/sirupsen/logrus"
+
 	"github.com/percona/percona-toolkit/src/go/lib/pginfo"
 	"github.com/percona/percona-toolkit/src/go/pt-pg-summary/templates"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-	log "github.com/sirupsen/logrus"
-
-	_ "github.com/lib/pq"
 )
 
 var (
 	Build     string = "2020-04-23" //nolint
 	Commit    string                //nolint
 	GoVersion string = "1.14.1"     //nolint
-	Version   string = "3.3.1"      //nolint
+	Version   string = "3.4.0"      //nolint
 )
 
 type connOpts struct {

src/go/pt-pg-summary/main_test.go

@@ -5,10 +5,10 @@ import (
 	"os"
 	"testing"
 
+	"github.com/sirupsen/logrus"
+
 	"github.com/percona/percona-toolkit/src/go/lib/pginfo"
 	"github.com/percona/percona-toolkit/src/go/pt-pg-summary/internal/tu"
-
-	"github.com/sirupsen/logrus"
 )
 
 type Test struct {

src/go/pt-secure-collect/collect.go

@@ -16,10 +16,11 @@ import (
 	"time"
 
 	shellwords "github.com/mattn/go-shellwords"
-	"github.com/percona/percona-toolkit/src/go/pt-secure-collect/sanitize"
-	"github.com/percona/percona-toolkit/src/go/pt-secure-collect/sanitize/util"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/percona/percona-toolkit/src/go/pt-secure-collect/sanitize"
+	"github.com/percona/percona-toolkit/src/go/pt-secure-collect/sanitize/util"
 )
 
 func collectData(opts *cliOptions) error {

src/go/pt-secure-collect/main.go

@@ -15,8 +15,8 @@ import (
 	"github.com/alecthomas/kingpin"
 	"github.com/go-ini/ini"
 	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 	log "github.com/sirupsen/logrus"
+	"github.com/sirupsen/logrus"
 	"golang.org/x/crypto/ssh/terminal"
 )
 
@@ -86,7 +86,7 @@ var (
 
 	Build     string = "2020-04-23" //nolint
 	GoVersion string = "1.14.1"     //nolint
-	Version   string = "3.3.1"      //nolint
+	Version   string = "3.4.0"      //nolint
 	Commit    string                //nolint
 )
 

src/go/pt-secure-collect/sanitize.go

@@ -3,9 +3,10 @@ package main
 import (
 	"os"
 
+	"github.com/pkg/errors"
+
 	"github.com/percona/percona-toolkit/src/go/pt-secure-collect/sanitize"
 	"github.com/percona/percona-toolkit/src/go/pt-secure-collect/sanitize/util"
-	"github.com/pkg/errors"
 )
 
 func sanitizeFile(opts *cliOptions) error {