diff --git a/go.mod b/go.mod index d8a0a613..3aa6d0f7 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/google/uuid v1.3.0 github.com/hashicorp/go-version v1.2.1-0.20190424083514-192140e6f3e6 github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c - github.com/kr/pretty v0.1.0 + github.com/kr/pretty v0.3.0 github.com/lib/pq v1.2.0 github.com/mattn/go-shellwords v1.0.6 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe @@ -19,7 +19,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/shirou/gopsutil v2.20.8+incompatible github.com/sirupsen/logrus v1.6.0 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.7.1 go.mongodb.org/mongo-driver v1.7.1 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 @@ -42,10 +42,11 @@ require ( github.com/json-iterator/go v1.1.11 // indirect github.com/klauspost/compress v1.10.10 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect - github.com/kr/text v0.1.0 // indirect + github.com/kr/text v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.6.1 // indirect github.com/smartystreets/goconvey v1.7.2 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.0.2 // indirect diff --git a/go.sum b/go.sum index 90edf12f..315829a8 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,7 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafo github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRYS3XieqF+Z9B9gNxo/eANAJCF2eiN4= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -113,9 +114,13 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -152,6 +157,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/shirou/gopsutil v2.20.8+incompatible h1:8c7Atn0FAUZJo+f4wYbN0iVpdWniCQk7IYwGtgdh1mY= github.com/shirou/gopsutil v2.20.8+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -176,6 +183,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= @@ -262,6 +271,7 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.18.6 h1:osqrAXbOQjkKIWDTjrqxWQ3w0GkKb1KA1XkUGHHYpeE= k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI= diff --git a/src/go/pt-mongodb-index-check/README.md b/src/go/pt-mongodb-index-check/README.md new file mode 100644 index 00000000..e4dae670 --- /dev/null +++ b/src/go/pt-mongodb-index-check/README.md @@ -0,0 +1,48 @@ +# pt-mongodb-index-check + + + +## Introduction + +This tool can perform checks on MongoDB indexes. + +Currently, these checks are available: + +### Duplicated indexes + +Check for indexes that are the prefix of other indexes. For example if we have these 2 indexes + +``` +db.getSiblingDB("testdb").test_col.createIndex({"f1": 1, "f2": -1, "f3": 1, "f4": 1}, {"name": "idx_01"}); +db.getSiblingDB("testdb").test_col.createIndex({"f1": 1, "f2": -1, "f3": 1}, {"name": "idx_02"}); +``` + +The index `idx_02` is the prefix of `idx_01` because it has the same keys in the same order so, `idx_02` can be dropped. + +### Unused indexes. + +This check gets the `$indexstats` for all indexes and reports those having `accesses.ops` = 0. + +## Usage + +Run the program as `pt-mongodb-index-check [flags]` + +#### Available commands + +| Command | Description | +| ---------------- | ---------------------------------- | +| check-duplicated | Run checks for duplicated indexes. | +| check-unused | Run check for unused indexes. | +| check-all | Run all checks | + +#### Available flags + +| Flag | Description | +| ---- | ----------- | +|--all-databases|Check in all databases excluding system dbs.| +|--databases=DATABASES,...|Comma separated list of databases to check.| +|--all-collections|Check in all collections in the selected databases.| +|--collections=COLLECTIONS,...|Comma separated list of collections to check.| +|--mongodb.uri=|Connection URI| +|--json|Show output as JSON| + diff --git a/src/go/pt-mongodb-index-check/indexes/duplicated.go b/src/go/pt-mongodb-index-check/indexes/duplicated.go index b69a9525..89742e22 100644 --- a/src/go/pt-mongodb-index-check/indexes/duplicated.go +++ b/src/go/pt-mongodb-index-check/indexes/duplicated.go @@ -20,13 +20,24 @@ type collectionIndex struct { func (di collectionIndex) ComparableKey() string { str := "" for _, elem := range di.Key { - sign := "+" + str += sign(elem) + elem.Key + } + return str +} + +func sign(elem primitive.E) string { + sign := "+" + switch elem.Value.(type) { + case int32: // internal MongoDB indexes like _id_ or lastUsed have the sign field as int32. if elem.Value.(int32) < 0 { sign = "-" } - str += sign + elem.Key + case float64: // All other indexes have the sign field as float64. + if elem.Value.(float64) < 0 { + sign = "-" + } } - return str + return sign } // IndexKey holds the list of fields that are part of an index, along with the field order. @@ -37,11 +48,7 @@ type IndexKey []primitive.E func (di IndexKey) String() string { str := "" for _, elem := range di { - sign := "+" - if elem.Value.(int32) < 0 { - sign = "-" - } - str += sign + elem.Key + " " + str += sign(elem) + elem.Key + " " } return str diff --git a/src/go/pt-mongodb-index-check/indexes/duplicated_test.go b/src/go/pt-mongodb-index-check/indexes/duplicated_test.go index e94f782f..708d7f02 100644 --- a/src/go/pt-mongodb-index-check/indexes/duplicated_test.go +++ b/src/go/pt-mongodb-index-check/indexes/duplicated_test.go @@ -62,9 +62,10 @@ func TestDuplicateIndexes(t *testing.T) { assert.Equal(t, 1, errCount) - want := []DuplicateIndex{ + want := []Duplicate{ { - Name: "idx_03", + Name: "idx_03", + Namespace: "test_db.test_col", Key: IndexKey{ {Key: "f1", Value: int32(1)}, {Key: "f2", Value: int32(-1)}, @@ -77,7 +78,8 @@ func TestDuplicateIndexes(t *testing.T) { }, }, { - Name: "idx_03", + Name: "idx_03", + Namespace: "test_db.test_col", Key: IndexKey{ {Key: "f1", Value: int32(1)}, {Key: "f2", Value: int32(-1)}, @@ -91,7 +93,8 @@ func TestDuplicateIndexes(t *testing.T) { }, }, { - Name: "idx_02", + Name: "idx_02", + Namespace: "test_db.test_col", Key: IndexKey{ {Key: "f1", Value: int32(1)}, {Key: "f2", Value: int32(-1)}, @@ -107,7 +110,7 @@ func TestDuplicateIndexes(t *testing.T) { }, } - di, err := FindDuplicatedIndexes(ctx, client, dbname, collname) + di, err := FindDuplicated(ctx, client, dbname, collname) assert.NoError(t, err) assert.Equal(t, want, di) } diff --git a/src/go/pt-mongodb-index-check/indexes/unused.go b/src/go/pt-mongodb-index-check/indexes/unused.go index a7d7a5dc..b3d69dd0 100644 --- a/src/go/pt-mongodb-index-check/indexes/unused.go +++ b/src/go/pt-mongodb-index-check/indexes/unused.go @@ -6,8 +6,10 @@ import ( "github.com/pkg/errors" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" + "gopkg.in/mgo.v2/bson" ) +var systemDBs = []string{"admin", "config", "local"} //nolint:gochecknoglobals // IndexStat hold an index usage statistics. type IndexStat struct { Accesses struct { @@ -25,11 +27,25 @@ type IndexStat struct { Host string `bson:"host"` } +func in(search string, items []string) bool { + for _, item := range items { + if search == item { + return true + } + } + return false +} + // FindUnusedIndexes returns a list of unused indexes for the given database and collection. -func FindUnusedIndexes(ctx context.Context, client *mongo.Client, database, collection string) ([]IndexStat, error) { +func FindUnused(ctx context.Context, client *mongo.Client, database, collection string) ([]IndexStat, error) { aggregation := mongo.Pipeline{ {{Key: "$indexStats", Value: primitive.M{}}}, {{Key: "$match", Value: primitive.M{"accesses.ops": 0}}}, + {{Key: "$match", Value: primitive.M{"name": bson.M{"$ne": "_id_"}}}}, + } + + if in(database, systemDBs) { + return nil, nil } cursor, err := client.Database(database).Collection(collection).Aggregate(ctx, aggregation) diff --git a/src/go/pt-mongodb-index-check/indexes/unused_test.go b/src/go/pt-mongodb-index-check/indexes/unused_test.go index 6f39268d..861d159a 100644 --- a/src/go/pt-mongodb-index-check/indexes/unused_test.go +++ b/src/go/pt-mongodb-index-check/indexes/unused_test.go @@ -59,9 +59,9 @@ func TestUnusedIndexes(t *testing.T) { assert.NoError(t, err) } - want := []string{"_id_", "idx_00", "idx_01", "idx_02"} + want := []string{"idx_00", "idx_01", "idx_02"} - ui, err := FindUnusedIndexes(ctx, client, dbname, collname) + ui, err := FindUnused(ctx, client, dbname, collname) assert.NoError(t, err) got := make([]string, 0, len(ui)) diff --git a/src/go/pt-mongodb-index-check/main.go b/src/go/pt-mongodb-index-check/main.go index f595d1d8..064c2c55 100644 --- a/src/go/pt-mongodb-index-check/main.go +++ b/src/go/pt-mongodb-index-check/main.go @@ -1,15 +1,22 @@ package main import ( + "bytes" "context" + "encoding/json" "fmt" - "log" + "text/template" "time" "github.com/alecthomas/kong" "github.com/percona/percona-toolkit/src/go/pt-mongodb-index-check/indexes" + "github.com/percona/percona-toolkit/src/go/pt-mongodb-index-check/templates" + "github.com/pkg/errors" + "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" + + log "github.com/sirupsen/logrus" ) type cmdlineArgs struct { @@ -24,6 +31,7 @@ type cmdlineArgs struct { AllCollections bool `name:"all-collections" xor:"colls" help:"Check in all collections in the selected databases."` Collections []string `name:"collections" xor:"colls" help:"Comma separated list of collections to check"` URI string `name:"mongodb.uri" help:"Connection URI"` + JSON bool `name:"json" help:"Show output as JSON"` } type response struct { @@ -43,18 +51,113 @@ func main() { log.Fatalf("Cannot connect to the database: %q", err) } + if args.AllDatabases { + args.Databases, err = client.ListDatabaseNames(context.TODO(), primitive.D{}) + if err != nil { + log.Fatalf("cannot list all databases: %s", err) + } + } + if args.AllCollections { + args.Collections = nil + } + resp := response{} switch kongctx.Command() { - case "list-unused": - for _, database := range args.Databases { - for _, collection := range args.Collections { - dups, err = indexes.FindDuplicated(ctx, client, database, collection) - } - } - fmt.Printf("databases: %v\n", args.Databases) - case "list-duplicates": + case "check-unused": + resp.Unused = append(resp.Unused, findUnused(ctx, client, args.Databases, args.Collections)...) + case "check-duplicates": + resp.Duplicated = append(resp.Duplicated, findDuplicated(ctx, client, args.Databases, args.Collections)...) + case "check-all": + resp.Unused = append(resp.Unused, findUnused(ctx, client, args.Databases, args.Collections)...) + resp.Duplicated = append(resp.Duplicated, findDuplicated(ctx, client, args.Databases, args.Collections)...) default: kong.DefaultHelpPrinter(kong.HelpOptions{}, kongctx) } + + fmt.Println(output(resp, args.JSON)) +} + +func output(resp response, asJson bool) string { + if asJson { + jsonStr, err := json.MarshalIndent(resp, "", "\t") + if err != nil { + log.Fatal("cannot encode the response as json") + } + return string(jsonStr) + } + + buf := new(bytes.Buffer) + + t := template.Must(template.New("duplicated").Parse(templates.Duplicated)) + if err := t.Execute(buf, resp.Duplicated); err != nil { + log.Fatal(errors.Wrap(err, "cannot parse clusterwide section of the output template")) + } + + t = template.Must(template.New("unused").Parse(templates.Unused)) + if err := t.Execute(buf, resp.Unused); err != nil { + log.Fatal(errors.Wrap(err, "cannot parse clusterwide section of the output template")) + } + + return buf.String() +} + +func findUnused(ctx context.Context, client *mongo.Client, databases []string, collections []string) []indexes.IndexStat { + unused := []indexes.IndexStat{} + var err error + + colls := make([]string, len(collections)) + copy(colls, collections) + + for _, database := range databases { + if len(collections) == 0 { + colls, err = client.Database(database).ListCollectionNames(ctx, primitive.D{}) + if err != nil { + log.Errorf("cannot get the list of collections for the database %s", database) + continue + } + } + + for _, collection := range colls { + idx, err := indexes.FindUnused(ctx, client, database, collection) + if err != nil { + log.Errorf("error while checking unused indexes in %s.%s: %s", database, collection, err) + continue + } + + unused = append(unused, idx...) + } + } + + return unused +} + +func findDuplicated(ctx context.Context, client *mongo.Client, databases []string, collections []string) []indexes.Duplicate { + duplicated := []indexes.Duplicate{} + var err error + + colls := make([]string, len(collections)) + copy(colls, collections) + + for _, database := range databases { + if len(collections) == 0 { + colls, err = client.Database(database).ListCollectionNames(ctx, primitive.D{}) + if err != nil { + log.Errorf("cannot get the list of collections for the database %s", database) + continue + } + } + + for _, collection := range colls { + dups, err := indexes.FindDuplicated(ctx, client, database, collection) + if err != nil { + log.Errorf("error while checking duplicated indexes in %s.%s: %s", database, collection, err) + continue + } + + duplicated = append(duplicated, dups...) + } + } + + return duplicated } diff --git a/src/go/pt-mongodb-index-check/templates/duplicated.go b/src/go/pt-mongodb-index-check/templates/duplicated.go new file mode 100644 index 00000000..2b2d1875 --- /dev/null +++ b/src/go/pt-mongodb-index-check/templates/duplicated.go @@ -0,0 +1,10 @@ +package templates + +// {{if $i}},{{end}} adds a comma after the first element. +// When $i == 0 (first element) {{ if $i }} returns false (0) + +var Duplicated = ` +Duplicated indexes +{{ range . }} +{{ .Namespace }}, index '{{ .Name }}', with fields { {{- range $i, $val := .Key }}{{if $i}}, {{end}}{{ $val.Key }}:{{ $val.Value }}{{ end -}} } is the prefix of '{{ .ContainerName }}' with fields { {{- range $i, $val := .ContainerKey }}{{if $i}}, {{end}}{{ $val.Key }}:{{ $val.Value }}{{ end -}} }{{ end}} +` diff --git a/src/go/pt-mongodb-index-check/templates/unused.go b/src/go/pt-mongodb-index-check/templates/unused.go new file mode 100644 index 00000000..ef06d283 --- /dev/null +++ b/src/go/pt-mongodb-index-check/templates/unused.go @@ -0,0 +1,10 @@ +package templates + +// {{if $i}},{{end}} adds a comma after the first element. +// When $i == 0 (first element) {{ if $i }} returns false (0) + +var Unused = ` +Unused indexes +{{ range . }} +{{ .Spec.Namespace }}, index '{{ .Name }}' with fields { {{- range $i, $val := .Key }}{{if $i}}, {{end}}{{ $val.Key }}:{{ $val.Value }} }{{ end }}{{ end}} +`