PT-1978 Implemented duplicated indexes detection

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-version v1.2.1-0.20190424083514-192140e6f3e6
 	github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c
+	github.com/kr/pretty v0.1.0
 	github.com/lib/pq v1.2.0
 	github.com/mattn/go-shellwords v1.0.6
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe
@@ -26,6 +27,7 @@ require (
 )
 
 require (
+	github.com/AlekSi/pointer v1.2.0 // indirect
 	github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 // indirect
@@ -39,6 +41,7 @@ require (
 	github.com/json-iterator/go v1.1.11 // indirect
 	github.com/klauspost/compress v1.10.10 // indirect
 	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
+	github.com/kr/text v0.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect

go.sum

@@ -1,3 +1,5 @@
+github.com/AlekSi/pointer v1.2.0 h1:glcy/gc4h8HnG2Z3ZECSzZ1IX1x2JxRVuDzaJwQE0+w=
+github.com/AlekSi/pointer v1.2.0/go.mod h1:gZGfd3dpW4vEc/UlyfKKi1roIqcCgwOIvb0tSNSBle0=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc=
 github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=

src/go/pt-mongodb-summary/indexes/duplicated_indexes.go

@@ -0,0 +1,88 @@
+package indexes
+
+import (
+	"context"
+	"log"
+	"sort"
+	"strings"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+type CollectionIndex struct {
+	Name      string      `bson:"name"`
+	Namespace string      `bson:"ns"`
+	V         int         `bson:"v"`
+	Key       primitive.D `bson:"key"`
+}
+
+func (di CollectionIndex) ComparableKey() string {
+	str := ""
+	for _, elem := range di.Key {
+		sign := "+"
+		if elem.Value.(int32) < 0 {
+			sign = "-"
+		}
+		str += sign + elem.Key
+	}
+	return str
+}
+
+type IndexKey []primitive.E
+
+func (di IndexKey) String() string {
+	str := ""
+	for _, elem := range di {
+		sign := "+"
+		if elem.Value.(int32) < 0 {
+			sign = "-"
+		}
+		str += sign + elem.Key + " "
+	}
+
+	return str
+}
+
+type DuplicateIndex struct {
+	Name          string
+	Key           IndexKey
+	ContainerName string
+	ContainerKey  IndexKey
+}
+
+func FindDuplicatedIndexes(ctx context.Context, client *mongo.Client, database, collection string) ([]DuplicateIndex, error) {
+	di := []DuplicateIndex{}
+
+	cursor, err := client.Database(database).Collection(collection).Indexes().List(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	var results []CollectionIndex
+	if err = cursor.All(context.TODO(), &results); err != nil {
+		log.Fatal(err)
+	}
+
+	sort.Slice(results, func(i, j int) bool {
+		return results[i].ComparableKey() < results[j].ComparableKey()
+	})
+
+	for i := 0; i < len(results)-1; i++ {
+		for j := i + 1; j < len(results); j++ {
+			if strings.HasPrefix(results[j].ComparableKey(), results[i].ComparableKey()) {
+				idx := DuplicateIndex{
+					Name:          results[i].Name,
+					Key:           make([]primitive.E, len(results[i].Key)),
+					ContainerName: results[j].Name,
+					ContainerKey:  make([]primitive.E, len(results[j].Key)),
+				}
+				copy(idx.Key, results[i].Key)
+				copy(idx.ContainerKey, results[j].Key)
+				di = append(di, idx)
+			}
+		}
+	}
+
+	return di, nil
+}

src/go/pt-mongodb-summary/indexes/duplicated_indexes_test.go

@@ -0,0 +1,113 @@
+package indexes
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/AlekSi/pointer"
+	tu "github.com/percona/percona-toolkit/src/go/internal/testutils"
+	"github.com/stretchr/testify/assert"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"gopkg.in/mgo.v2/bson"
+)
+
+func TestDuplicateIndexes(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	client, err := tu.TestClient(ctx, tu.MongoDBShard1PrimaryPort)
+	if err != nil {
+		t.Fatalf("cannot get a new MongoDB client: %s", err)
+	}
+
+	dbname := "test_db"
+	collname := "test_col"
+
+	database := client.Database(dbname)
+	database.Drop(ctx)       //nolint:errcheck
+	defer database.Drop(ctx) //nolint:errcheck
+
+	_, err = database.Collection(collname).InsertOne(ctx, bson.M{"f1": 1, "f2": "2", "f3": "a", "f4": "c"})
+	assert.NoError(t, err)
+
+	testCases := []primitive.D{
+		{{"f1", 1}, {"f2", -1}, {"f3", 1}, {"f4", 1}},
+		{{"f1", 1}, {"f2", -1}, {"f3", 1}, {"f4", 1}}, // this will throw a duplicate index error
+		{{"f1", 1}, {"f2", -1}, {"f3", 1}},
+		{{"f1", 1}, {"f2", -1}},
+		{{"f3", -1}},
+	}
+
+	errCount := 0
+	for i, tc := range testCases {
+		mod := mongo.IndexModel{
+			Keys: tc,
+			Options: &options.IndexOptions{
+				Name: pointer.ToString(fmt.Sprintf("idx_%02d", i)),
+			},
+		}
+		_, err := database.Collection(collname).Indexes().CreateOne(ctx, mod)
+		if err != nil {
+			errCount++
+		}
+	}
+	/*
+	 At this point we have 5 indexes: _id: (MongoDB's default), idx_00, idx_02, idx_03, idx_04.
+	 idx_01 wasn't created since it duplicates idx_00 and errCount=1.
+	*/
+
+	assert.Equal(t, 1, errCount)
+
+	want := []DuplicateIndex{
+		{
+			Name: "idx_03",
+			Key: IndexKey{
+				{Key: "f1", Value: int32(1)},
+				{Key: "f2", Value: int32(-1)},
+			},
+			ContainerName: "idx_02",
+			ContainerKey: IndexKey{
+				{Key: "f1", Value: int32(1)},
+				{Key: "f2", Value: int32(-1)},
+				{Key: "f3", Value: int32(1)},
+			},
+		},
+		{
+			Name: "idx_03",
+			Key: IndexKey{
+				{Key: "f1", Value: int32(1)},
+				{Key: "f2", Value: int32(-1)},
+			},
+			ContainerName: "idx_00",
+			ContainerKey: IndexKey{
+				{Key: "f1", Value: int32(1)},
+				{Key: "f2", Value: int32(-1)},
+				{Key: "f3", Value: int32(1)},
+				{Key: "f4", Value: int32(1)},
+			},
+		},
+		{
+			Name: "idx_02",
+			Key: IndexKey{
+				{Key: "f1", Value: int32(1)},
+				{Key: "f2", Value: int32(-1)},
+				{Key: "f3", Value: int32(1)},
+			},
+			ContainerName: "idx_00",
+			ContainerKey: IndexKey{
+				{Key: "f1", Value: int32(1)},
+				{Key: "f2", Value: int32(-1)},
+				{Key: "f3", Value: int32(1)},
+				{Key: "f4", Value: int32(1)},
+			},
+		},
+	}
+
+	di, err := FindDuplicatedIndexes(ctx, client, dbname, collname)
+	assert.NoError(t, err)
+	assert.Equal(t, want, di)
+}