PT-1978 New tool pt-mongodb-check-index

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-version v1.2.1-0.20190424083514-192140e6f3e6
 	github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c
-	github.com/kr/pretty v0.1.0
+	github.com/kr/pretty v0.3.0
 	github.com/lib/pq v1.2.0
 	github.com/mattn/go-shellwords v1.0.6
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe
@@ -19,7 +19,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/shirou/gopsutil v2.20.8+incompatible
 	github.com/sirupsen/logrus v1.6.0
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.7.1
 	go.mongodb.org/mongo-driver v1.7.1
 	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22
@@ -42,10 +42,11 @@ require (
 	github.com/json-iterator/go v1.1.11 // indirect
 	github.com/klauspost/compress v1.10.10 // indirect
 	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
-	github.com/kr/text v0.1.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rogpeppe/go-internal v1.6.1 // indirect
 	github.com/smartystreets/goconvey v1.7.2 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.0.2 // indirect

go.sum

@@ -17,6 +17,7 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafo
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRYS3XieqF+Z9B9gNxo/eANAJCF2eiN4=
 github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -113,9 +114,13 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJ
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -152,6 +157,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/shirou/gopsutil v2.20.8+incompatible h1:8c7Atn0FAUZJo+f4wYbN0iVpdWniCQk7IYwGtgdh1mY=
 github.com/shirou/gopsutil v2.20.8+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
@@ -176,6 +183,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
@@ -262,6 +271,7 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 k8s.io/api v0.18.6 h1:osqrAXbOQjkKIWDTjrqxWQ3w0GkKb1KA1XkUGHHYpeE=
 k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI=

src/go/pt-mongodb-index-check/README.md

@@ -0,0 +1,48 @@
+# pt-mongodb-index-check
+
+
+
+## Introduction
+
+This tool can perform checks on MongoDB indexes. 
+
+Currently, these checks are available:
+
+### Duplicated indexes
+
+Check for indexes that are the prefix of other indexes. For example if we have these 2 indexes
+
+```
+db.getSiblingDB("testdb").test_col.createIndex({"f1": 1, "f2": -1, "f3": 1, "f4": 1}, {"name": "idx_01"});
+db.getSiblingDB("testdb").test_col.createIndex({"f1": 1, "f2": -1, "f3": 1}, {"name": "idx_02"});
+```
+
+The index `idx_02` is the prefix of `idx_01` because it has the same keys in the same order so, `idx_02` can be dropped.
+
+### Unused indexes.
+
+This check gets the `$indexstats` for all indexes and reports those having `accesses.ops` = 0.
+
+## Usage
+
+Run the program as `pt-mongodb-index-check <command> [flags]`
+
+#### Available commands
+
+| Command          | Description                        |
+| ---------------- | ---------------------------------- |
+| check-duplicated | Run checks for duplicated indexes. |
+| check-unused     | Run check for unused indexes.      |
+| check-all        | Run all checks                     |
+
+#### Available flags
+
+| Flag | Description |
+| ---- | ----------- |
+|--all-databases|Check in all databases excluding system dbs.|
+|--databases=DATABASES,...|Comma separated list of databases to check.|
+|--all-collections|Check in all collections in the selected databases.|
+|--collections=COLLECTIONS,...|Comma separated list of collections to check.|
+|--mongodb.uri=<connection string>|Connection URI|
+|--json|Show output as JSON|
+

src/go/pt-mongodb-index-check/indexes/duplicated.go

@@ -20,13 +20,24 @@ type collectionIndex struct {
 func (di collectionIndex) ComparableKey() string {
 	str := ""
 	for _, elem := range di.Key {
-		sign := "+"
+		str += sign(elem) + elem.Key
+	}
+	return str
+}
+
+func sign(elem primitive.E) string {
+	sign := "+"
+	switch elem.Value.(type) {
+	case int32: // internal MongoDB indexes like _id_ or lastUsed have the sign field as int32.
 		if elem.Value.(int32) < 0 {
 			sign = "-"
 		}
-		str += sign + elem.Key
+	case float64: // All other indexes have the sign field as float64.
+		if elem.Value.(float64) < 0 {
+			sign = "-"
+		}
 	}
-	return str
+	return sign
 }
 
 // IndexKey holds the list of fields that are part of an index, along with the field order.
@@ -37,11 +48,7 @@ type IndexKey []primitive.E
 func (di IndexKey) String() string {
 	str := ""
 	for _, elem := range di {
-		sign := "+"
-		if elem.Value.(int32) < 0 {
-			sign = "-"
-		}
-		str += sign + elem.Key + " "
+		str += sign(elem) + elem.Key + " "
 	}
 
 	return str

src/go/pt-mongodb-index-check/indexes/duplicated_test.go

@@ -62,9 +62,10 @@ func TestDuplicateIndexes(t *testing.T) {
 
 	assert.Equal(t, 1, errCount)
 
-	want := []DuplicateIndex{
+	want := []Duplicate{
 		{
-			Name: "idx_03",
+			Name:      "idx_03",
+			Namespace: "test_db.test_col",
 			Key: IndexKey{
 				{Key: "f1", Value: int32(1)},
 				{Key: "f2", Value: int32(-1)},
@@ -77,7 +78,8 @@ func TestDuplicateIndexes(t *testing.T) {
 			},
 		},
 		{
-			Name: "idx_03",
+			Name:      "idx_03",
+			Namespace: "test_db.test_col",
 			Key: IndexKey{
 				{Key: "f1", Value: int32(1)},
 				{Key: "f2", Value: int32(-1)},
@@ -91,7 +93,8 @@ func TestDuplicateIndexes(t *testing.T) {
 			},
 		},
 		{
-			Name: "idx_02",
+			Name:      "idx_02",
+			Namespace: "test_db.test_col",
 			Key: IndexKey{
 				{Key: "f1", Value: int32(1)},
 				{Key: "f2", Value: int32(-1)},
@@ -107,7 +110,7 @@ func TestDuplicateIndexes(t *testing.T) {
 		},
 	}
 
-	di, err := FindDuplicatedIndexes(ctx, client, dbname, collname)
+	di, err := FindDuplicated(ctx, client, dbname, collname)
 	assert.NoError(t, err)
 	assert.Equal(t, want, di)
 }

src/go/pt-mongodb-index-check/indexes/unused.go

@@ -6,8 +6,10 @@ import (
 	"github.com/pkg/errors"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo"
+	"gopkg.in/mgo.v2/bson"
 )
 
+var systemDBs = []string{"admin", "config", "local"} //nolint:gochecknoglobals
 // IndexStat hold an index usage statistics.
 type IndexStat struct {
 	Accesses struct {
@@ -25,11 +27,25 @@ type IndexStat struct {
 	Host string      `bson:"host"`
 }
 
+func in(search string, items []string) bool {
+	for _, item := range items {
+		if search == item {
+			return true
+		}
+	}
+	return false
+}
+
 // FindUnusedIndexes returns a list of unused indexes for the given database and collection.
-func FindUnusedIndexes(ctx context.Context, client *mongo.Client, database, collection string) ([]IndexStat, error) {
+func FindUnused(ctx context.Context, client *mongo.Client, database, collection string) ([]IndexStat, error) {
 	aggregation := mongo.Pipeline{
 		{{Key: "$indexStats", Value: primitive.M{}}},
 		{{Key: "$match", Value: primitive.M{"accesses.ops": 0}}},
+		{{Key: "$match", Value: primitive.M{"name": bson.M{"$ne": "_id_"}}}},
+	}
+
+	if in(database, systemDBs) {
+		return nil, nil
 	}
 
 	cursor, err := client.Database(database).Collection(collection).Aggregate(ctx, aggregation)

src/go/pt-mongodb-index-check/indexes/unused_test.go

@@ -59,9 +59,9 @@ func TestUnusedIndexes(t *testing.T) {
 		assert.NoError(t, err)
 	}
 
-	want := []string{"_id_", "idx_00", "idx_01", "idx_02"}
+	want := []string{"idx_00", "idx_01", "idx_02"}
 
-	ui, err := FindUnusedIndexes(ctx, client, dbname, collname)
+	ui, err := FindUnused(ctx, client, dbname, collname)
 	assert.NoError(t, err)
 
 	got := make([]string, 0, len(ui))

src/go/pt-mongodb-index-check/main.go

@@ -1,15 +1,22 @@
 package main
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
-	"log"
+	"text/template"
 	"time"
 
 	"github.com/alecthomas/kong"
 	"github.com/percona/percona-toolkit/src/go/pt-mongodb-index-check/indexes"
+	"github.com/percona/percona-toolkit/src/go/pt-mongodb-index-check/templates"
+	"github.com/pkg/errors"
+	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
+
+	log "github.com/sirupsen/logrus"
 )
 
 type cmdlineArgs struct {
@@ -24,6 +31,7 @@ type cmdlineArgs struct {
 	AllCollections bool     `name:"all-collections" xor:"colls" help:"Check in all collections in the selected databases."`
 	Collections    []string `name:"collections" xor:"colls" help:"Comma separated list of collections to check"`
 	URI            string   `name:"mongodb.uri" help:"Connection URI"`
+	JSON           bool     `name:"json" help:"Show output as JSON"`
 }
 
 type response struct {
@@ -43,18 +51,113 @@ func main() {
 		log.Fatalf("Cannot connect to the database: %q", err)
 	}
 
+	if args.AllDatabases {
+		args.Databases, err = client.ListDatabaseNames(context.TODO(), primitive.D{})
+		if err != nil {
+			log.Fatalf("cannot list all databases: %s", err)
+		}
+	}
+	if args.AllCollections {
+		args.Collections = nil
+	}
+
 	resp := response{}
 
 	switch kongctx.Command() {
-	case "list-unused":
-		for _, database := range args.Databases {
-			for _, collection := range args.Collections {
-				dups, err = indexes.FindDuplicated(ctx, client, database, collection)
-			}
-		}
-		fmt.Printf("databases: %v\n", args.Databases)
-	case "list-duplicates":
+	case "check-unused":
+		resp.Unused = append(resp.Unused, findUnused(ctx, client, args.Databases, args.Collections)...)
+	case "check-duplicates":
+		resp.Duplicated = append(resp.Duplicated, findDuplicated(ctx, client, args.Databases, args.Collections)...)
+	case "check-all":
+		resp.Unused = append(resp.Unused, findUnused(ctx, client, args.Databases, args.Collections)...)
+		resp.Duplicated = append(resp.Duplicated, findDuplicated(ctx, client, args.Databases, args.Collections)...)
 	default:
 		kong.DefaultHelpPrinter(kong.HelpOptions{}, kongctx)
 	}
+
+	fmt.Println(output(resp, args.JSON))
+}
+
+func output(resp response, asJson bool) string {
+	if asJson {
+		jsonStr, err := json.MarshalIndent(resp, "", "\t")
+		if err != nil {
+			log.Fatal("cannot encode the response as json")
+		}
+		return string(jsonStr)
+	}
+
+	buf := new(bytes.Buffer)
+
+	t := template.Must(template.New("duplicated").Parse(templates.Duplicated))
+	if err := t.Execute(buf, resp.Duplicated); err != nil {
+		log.Fatal(errors.Wrap(err, "cannot parse clusterwide section of the output template"))
+	}
+
+	t = template.Must(template.New("unused").Parse(templates.Unused))
+	if err := t.Execute(buf, resp.Unused); err != nil {
+		log.Fatal(errors.Wrap(err, "cannot parse clusterwide section of the output template"))
+	}
+
+	return buf.String()
+}
+
+func findUnused(ctx context.Context, client *mongo.Client, databases []string, collections []string) []indexes.IndexStat {
+	unused := []indexes.IndexStat{}
+	var err error
+
+	colls := make([]string, len(collections))
+	copy(colls, collections)
+
+	for _, database := range databases {
+		if len(collections) == 0 {
+			colls, err = client.Database(database).ListCollectionNames(ctx, primitive.D{})
+			if err != nil {
+				log.Errorf("cannot get the list of collections for the database %s", database)
+				continue
+			}
+		}
+
+		for _, collection := range colls {
+			idx, err := indexes.FindUnused(ctx, client, database, collection)
+			if err != nil {
+				log.Errorf("error while checking unused indexes in %s.%s: %s", database, collection, err)
+				continue
+			}
+
+			unused = append(unused, idx...)
+		}
+	}
+
+	return unused
+}
+
+func findDuplicated(ctx context.Context, client *mongo.Client, databases []string, collections []string) []indexes.Duplicate {
+	duplicated := []indexes.Duplicate{}
+	var err error
+
+	colls := make([]string, len(collections))
+	copy(colls, collections)
+
+	for _, database := range databases {
+		if len(collections) == 0 {
+			colls, err = client.Database(database).ListCollectionNames(ctx, primitive.D{})
+			if err != nil {
+				log.Errorf("cannot get the list of collections for the database %s", database)
+				continue
+			}
+		}
+
+		for _, collection := range colls {
+			dups, err := indexes.FindDuplicated(ctx, client, database, collection)
+			if err != nil {
+				log.Errorf("error while checking duplicated indexes in %s.%s: %s", database, collection, err)
+				continue
+			}
+
+			duplicated = append(duplicated, dups...)
+		}
+	}
+
+	return duplicated
 }

src/go/pt-mongodb-index-check/templates/duplicated.go

@@ -0,0 +1,10 @@
+package templates
+
+// {{if $i}},{{end}} adds a comma after the first element.
+// When $i == 0 (first element) {{ if $i }} returns false (0)
+
+var Duplicated = `
+Duplicated indexes 
+{{ range . }}
+{{ .Namespace }}, index '{{ .Name }}', with fields { {{- range $i, $val := .Key }}{{if $i}}, {{end}}{{ $val.Key }}:{{ $val.Value }}{{ end -}} } is the prefix of '{{ .ContainerName }}' with fields { {{- range $i, $val := .ContainerKey }}{{if $i}}, {{end}}{{ $val.Key }}:{{ $val.Value }}{{ end -}} }{{ end}}
+`

src/go/pt-mongodb-index-check/templates/unused.go

@@ -0,0 +1,10 @@
+package templates
+
+// {{if $i}},{{end}} adds a comma after the first element.
+// When $i == 0 (first element) {{ if $i }} returns false (0)
+
+var Unused = `
+Unused indexes 
+{{ range . }}
+{{ .Spec.Namespace }}, index '{{ .Name }}' with fields { {{- range $i, $val := .Key }}{{if $i}}, {{end}}{{ $val.Key }}:{{ $val.Value }} }{{ end }}{{ end}}
+`