diff --git a/Gopkg.lock b/Gopkg.lock index b8012e32..dca5c656 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -27,30 +27,30 @@ [[projects]] branch = "master" - digest = "1:a74730e052a45a3fab1d310fdef2ec17ae3d6af16228421e238320846f2aaec8" + digest = "1:a7f619ffc7b99687f9444bd0a07509fec8ae708a7175d234878129896c0918a8" name = "github.com/alecthomas/template" packages = [ ".", "parse", ] pruneopts = "" - revision = "a0175ee3bccc567396460bf5acd36800cb10c49c" + revision = "fb15b899a75114aa79cc930e33c46b577cc664b1" [[projects]] branch = "master" - digest = "1:8483994d21404c8a1d489f6be756e25bfccd3b45d65821f25695577791a08e68" + digest = "1:86ea497979270399c5e544bbdc6f3a846078cd7c6d9be40fd0012ab54fee4052" name = "github.com/alecthomas/units" packages = ["."] pruneopts = "" - revision = "2efee857e7cfd4f3d0138cc3cbb1b4966962b93a" + revision = "c3de453c63f4bdb4dadffab9805ec00426c505f7" [[projects]] - digest = "1:03edf882162b807cdf1bc558c66226167fa2f8eb44359eac2eeb3794a91cb168" + digest = "1:c3f4f5bc77b998746a8061a52a2b8fb172c1744801165227692d702483995d58" name = "github.com/go-ini/ini" packages = ["."] pruneopts = "" - revision = "c85607071cf08ca1adaf48319cd1aa322e81d8c1" - version = "v1.42.0" + revision = "d4cae42d398bc0095297fc3315669590d29166ea" + version = "v1.46.0" [[projects]] digest = "1:b6581f9180e0f2d5549280d71819ab951db9d511478c87daca95669589d505c0" @@ -72,20 +72,28 @@ version = "v1.8.0" [[projects]] - digest = "1:530233672f656641b365f8efb38ed9fba80e420baff2ce87633813ab3755ed6d" + digest = "1:68c64bb61d55dcd17c82ca0b871ddddb5ae18b30cfe26f6bfd4b6df6287dc2e0" name = "github.com/golang/mock" packages = ["gomock"] pruneopts = "" - revision = "51421b967af1f557f93a59e0057aaf15ca02e29c" - version = "v1.2.0" + revision = "9fa652df1129bef0e734c9cf9bf6dbae9ef3b9fa" + version = "1.3.1" [[projects]] branch = "master" - digest = "1:b759103c9b4135568253c17d2866064cde398e93764b611caabf5aa8e3059685" + digest = "1:6a6322a15aa8e99bd156fbba0aae4e5d67b4bb05251d860b348a45dfdcba9cce" + name = "github.com/golang/snappy" + packages = ["."] + pruneopts = "" + revision = "2a8bb927dd31d8daada140a5d09578521ce5c36a" + +[[projects]] + branch = "master" + digest = "1:4fbfcfe715329e2f09ea644657aa791b81e73a0c835a4f79b9a0dfff0513e2b8" name = "github.com/hashicorp/go-version" packages = ["."] pruneopts = "" - revision = "d40cf49b3a77bba84a7afdbd7f1dc295d114efb1" + revision = "192140e6f3e645d971b134d4e35b5191adb9dfd3" [[projects]] branch = "master" @@ -137,30 +145,24 @@ [[projects]] branch = "master" - digest = "1:020f67c818cb9c3fdc77d92c5744fb2d5b90930280cc43311ba43c6459fd0b98" + digest = "1:78d0c21e99e070382e26f862c80a9354593880d51461a4aeacd490f52d6e5942" name = "github.com/pborman/getopt" - packages = [ - ".", - "v2", - ] + packages = ["."] pruneopts = "" - revision = "fd6d657c3083960b8d604310c34a621ec24bdc6a" + revision = "ee0cd42419d3adee9239dbd1c375717fe482dac7" [[projects]] branch = "master" - digest = "1:7a840dbacabd648e5b511010dea5da9eed99030dd185b3c7c7195fdadb3051a8" + digest = "1:1adb91baf59317a1614c3b465b2734066c256d1dca99d5526baa43274542a737" name = "github.com/percona/go-mysql" packages = ["query"] pruneopts = "" - revision = "f5cfaf6a5e55b754b7b106f4488e1bc24cb8c2d6" + revision = "c5d0b4a3add9c9bf5bb26b2ab823289e395a3f98" [[projects]] digest = "1:16b4510ba61ab0bb7a4e694ea6396a7b2879f5fabb21e93066e182691f790173" name = "github.com/percona/pmgo" - packages = [ - ".", - "pmgomock", - ] + packages = ["."] pruneopts = "" revision = "497d06e28f910fbe26d5d60f59d36284a6901c6f" version = "0.5.2" @@ -182,19 +184,18 @@ version = "v1.2.0" [[projects]] - digest = "1:d77a85cf43b70ae61fa2543d402d782b40dca0f5f41413839b5f916782b0fab9" + digest = "1:2226ffdae873216a5bc8a0bab7a51ac670b27a4aed852007d77600f809aa04e3" name = "github.com/shirou/gopsutil" packages = [ "cpu", - "host", "internal/common", "mem", "net", "process", ] pruneopts = "" - revision = "6c6abd6d1666d6b27f1c261e0f850441ba22aa3a" - version = "v2.19.02" + revision = "d80c43f9c984a48783daf22f4bd9278006ae483a" + version = "v2.19.7" [[projects]] branch = "master" @@ -205,15 +206,31 @@ revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b" [[projects]] - digest = "1:b73fe282e350b3ef2c71d8ff08e929e0b9670b1bb5b7fde1d3c1b4cd6e6dc8b1" + digest = "1:1a405cddcf3368445051fb70ab465ae99da56ad7be8d8ca7fc52159d1c2d873c" name = "github.com/sirupsen/logrus" packages = ["."] pruneopts = "" - revision = "dae0fa8d5b0c810a8ab733fbd5510c7cae84eca4" - version = "v1.4.0" + revision = "839c75faf7f98a33d445d181f3018b5c3409a45e" + version = "v1.4.2" [[projects]] - digest = "1:52376909e3b93cfc66ff439bdf9a7bf88783a0017ceded23c3f1c76b9f7e10ee" + branch = "master" + digest = "1:ad74f33a69bd6ab0bd7287003b7c1069b94cfb5213eb5597005fe2963d7dfca9" + name = "github.com/xdg/scram" + packages = ["."] + pruneopts = "" + revision = "7eeb5667e42c09cb51bf7b7c28aea8c56767da90" + +[[projects]] + branch = "master" + digest = "1:62f6eb06f6f39d18fc961956116a50a0e52b89961ed8a83bbb950b4cfb09fe67" + name = "github.com/xdg/stringprep" + packages = ["."] + pruneopts = "" + revision = "73f8eece6fdcd902c185bf651de50f3828bed5ed" + +[[projects]] + digest = "1:5a7738096093da28b02967e9f29b380341c02baa4dc3104731a62be4290369b7" name = "go.mongodb.org/mongo-driver" packages = [ "bson", @@ -221,38 +238,88 @@ "bson/bsonrw", "bson/bsontype", "bson/primitive", + "event", + "internal", + "mongo", + "mongo/options", + "mongo/readconcern", + "mongo/readpref", + "mongo/writeconcern", + "tag", + "version", + "x/bsonx", "x/bsonx/bsoncore", + "x/mongo/driver", + "x/mongo/driver/auth", + "x/mongo/driver/auth/internal/gssapi", + "x/mongo/driver/session", + "x/mongo/driver/topology", + "x/mongo/driver/uuid", + "x/network/address", + "x/network/command", + "x/network/compressor", + "x/network/connection", + "x/network/connstring", + "x/network/description", + "x/network/result", + "x/network/wiremessage", ] pruneopts = "" - revision = "582ff343271e8893d785ff094855498c285bce0a" - version = "v1.0.3" + revision = "0d1270edf53072da4da781b76d2e1db58831152f" + version = "v1.0.4" [[projects]] branch = "master" - digest = "1:36ef1d8645934b1744cc7d8726e00d3dd9d8d84c18617bf7367a3a6d532f3370" + digest = "1:086760278d762dbb0e9a26e09b57f04c89178c86467d8d94fae47d64c222f328" name = "golang.org/x/crypto" - packages = ["ssh/terminal"] + packages = [ + "pbkdf2", + "ssh/terminal", + ] pruneopts = "" - revision = "a5d413f7728c81fb97d96a2b722368945f651e78" + revision = "4def268fd1a49955bfb3dda92fe3db4f924f2285" [[projects]] branch = "master" - digest = "1:adcb9e84ce154ef1d45851b57c40f8a211db3e36373a65b7c4f10c79b7428718" + digest = "1:955694a7c42527d7fb188505a22f10b3e158c6c2cf31fe64b1e62c9ab7b18401" name = "golang.org/x/net" packages = ["context"] pruneopts = "" - revision = "74de082e2cca95839e88aa0aeee5aadf6ce7710f" + revision = "ca1201d0de80cfde86cb01aea620983605dfe99b" [[projects]] branch = "master" - digest = "1:1b0de777d8ddd63356d5a4d76799ea8f47e811aa9dda85ddc72b2a061c799cc9" + digest = "1:9f6efefb4e401a4f699a295d14518871368eb89403f2dd23ec11dfcd2c0836ba" + name = "golang.org/x/sync" + packages = ["semaphore"] + pruneopts = "" + revision = "112230192c580c3556b8cee6403af37a4fc5f28c" + +[[projects]] + branch = "master" + digest = "1:0b5c2207c72f2d13995040f176feb6e3f453d6b01af2b9d57df76b05ded2e926" name = "golang.org/x/sys" packages = [ "unix", "windows", ] pruneopts = "" - revision = "9eb1bfa1ce65ae8a6ff3114b0aaf9a41a6cf3560" + revision = "51ab0e2deafac1f46c46ad59cf0921be2f180c3d" + +[[projects]] + digest = "1:740b51a55815493a8d0f2b1e0d0ae48fe48953bf7eaf3fcc4198823bf67768c0" + name = "golang.org/x/text" + packages = [ + "internal/gen", + "internal/triegen", + "internal/ucd", + "transform", + "unicode/cldr", + "unicode/norm", + ] + pruneopts = "" + revision = "342b2e1fbaa52c93f31447ad2c6abc048c63e475" + version = "v0.3.2" [[projects]] branch = "v2" @@ -291,20 +358,17 @@ "github.com/mattn/go-shellwords", "github.com/montanaflynn/stats", "github.com/pborman/getopt", - "github.com/pborman/getopt/v2", "github.com/percona/go-mysql/query", "github.com/percona/pmgo", - "github.com/percona/pmgo/pmgomock", "github.com/pkg/errors", "github.com/satori/go.uuid", "github.com/shirou/gopsutil/process", "github.com/sirupsen/logrus", "go.mongodb.org/mongo-driver/bson", "go.mongodb.org/mongo-driver/bson/primitive", + "go.mongodb.org/mongo-driver/mongo", + "go.mongodb.org/mongo-driver/mongo/options", "golang.org/x/crypto/ssh/terminal", - "gopkg.in/mgo.v2", - "gopkg.in/mgo.v2/bson", - "gopkg.in/mgo.v2/dbtest", ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/src/go/.env b/src/go/.env index 357131de..d15d4000 100644 --- a/src/go/.env +++ b/src/go/.env @@ -1,7 +1,7 @@ AWS_ACCESS_KEY_ID=AKIARXP3OARBNV35P5M5 AWS_SECRET_ACCESS_KEY=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 GOCACHE= -GOLANG_DOCKERHUB_TAG= +GOLANG_DOCKERHUB_TAG=1.10-stretch TEST_MONGODB_ADMIN_USERNAME=admin TEST_MONGODB_ADMIN_PASSWORD=admin123456 TEST_MONGODB_USERNAME=test @@ -24,5 +24,5 @@ TEST_MONGODB_CONFIGSVR1_PORT=17007 TEST_MONGODB_CONFIGSVR2_PORT=17008 TEST_MONGODB_CONFIGSVR3_PORT=17009 TEST_MONGODB_MONGOS_PORT=17000 -TEST_PSMDB_VERSION=3.6 -TEST_MONGODB_FLAVOR=percona/percona-server-mongodb +TEST_PSMDB_VERSION=4.0 +TEST_MONGODB_FLAVOR=mongo diff --git a/src/go/lib/profiling/profiling.go b/src/go/lib/profiling/profiling.go index 40ef4cb3..cf69cc22 100644 --- a/src/go/lib/profiling/profiling.go +++ b/src/go/lib/profiling/profiling.go @@ -18,77 +18,22 @@ package profiling import ( - "github.com/percona/pmgo" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" + "context" + + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" ) -func Enable(url string) error { - session, err := createSession(url) - if err != nil { - return err - } - defer session.Close() - - err = profile(session.DB(""), 2) - if err != nil { - return err - } - - return nil +func Enable(ctx context.Context, client *mongo.Client) error { + res := client.Database("admin").RunCommand(ctx, primitive.M{"profile": 2}) + return res.Err() } -func Disable(url string) error { - session, err := createSession(url) - if err != nil { - return err - } - defer session.Close() - - err = profile(session.DB(""), 0) - if err != nil { - return err - } - - return nil +func Disable(ctx context.Context, client *mongo.Client) error { + res := client.Database("admin").RunCommand(ctx, primitive.M{"profile": 0}) + return res.Err() } -func Drop(url string) error { - session, err := createSession(url) - if err != nil { - return err - } - defer session.Close() - - return session.DB("").C("system.profile").DropCollection() -} - -func profile(db pmgo.DatabaseManager, v int) error { - result := struct { - Was int - Slowms int - Ratelimit int - }{} - return db.Run( - bson.M{ - "profile": v, - }, - &result, - ) -} - -func createSession(url string) (pmgo.SessionManager, error) { - dialInfo, err := pmgo.ParseURL(url) - if err != nil { - return nil, err - } - dialer := pmgo.NewDialer() - - session, err := dialer.DialWithInfo(dialInfo) - if err != nil { - return nil, err - } - - session.SetMode(mgo.Eventual, true) - return session, nil +func Drop(ctx context.Context, client *mongo.Client) error { + return client.Database("").Collection("system.profile").Drop(ctx) } diff --git a/src/go/lib/tutil/util.go b/src/go/lib/tutil/util.go index e450859c..8f42a713 100644 --- a/src/go/lib/tutil/util.go +++ b/src/go/lib/tutil/util.go @@ -8,7 +8,7 @@ import ( "regexp" "strings" - "gopkg.in/mgo.v2/bson" + "go.mongodb.org/mongo-driver/bson" ) const ( @@ -77,7 +77,7 @@ func LoadBson(filename string, destination interface{}) error { re = regexp.MustCompile(`(?s): (function \(.*?\) {.*?})`) buf = re.ReplaceAll(buf, []byte(`: ""`)) - err = bson.UnmarshalJSON(buf, &destination) + err = bson.Unmarshal(buf, &destination) if err != nil { return err } diff --git a/src/go/mongolib/fingerprinter/fingerprinter_test.go b/src/go/mongolib/fingerprinter/fingerprinter_test.go index a42e477c..300ff7cb 100644 --- a/src/go/mongolib/fingerprinter/fingerprinter_test.go +++ b/src/go/mongolib/fingerprinter/fingerprinter_test.go @@ -10,7 +10,7 @@ import ( "github.com/percona/percona-toolkit/src/go/lib/tutil" "github.com/percona/percona-toolkit/src/go/mongolib/proto" - "gopkg.in/mgo.v2/bson" + "go.mongodb.org/mongo-driver/bson" ) const ( diff --git a/src/go/mongolib/proto/main_test.go b/src/go/mongolib/proto/main_test.go deleted file mode 100644 index bc614127..00000000 --- a/src/go/mongolib/proto/main_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package proto_test - -import ( - "fmt" - "io/ioutil" - "os" - "testing" - - "go.mongodb.org/mongo-driver/bson" - mgo "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/dbtest" -) - -var Server dbtest.DBServer -var session *mgo.Session - -func TestMain(m *testing.M) { - // The tempdir is created so MongoDB has a location to store its files. - // Contents are wiped once the server stops - os.Setenv("CHECK_SESSIONS", "0") - tempDir, _ := ioutil.TempDir("", "testing") - Server.SetPath(tempDir) - session = Server.Session() - - retCode := m.Run() - - Server.Session().Close() - Server.Wipe() - - // Stop shuts down the temporary server and removes data on disk. - Server.Stop() - - // call with result of m.Run() - os.Exit(retCode) -} - -func ExamplePing() { - ss := map[string]interface{}{} - if err := session.DB("admin").Run(bson.D{{"ping", 1}}, &ss); err != nil { - panic(err) - } - fmt.Printf("%+v", ss) - // Output: map[ok:1] -} diff --git a/src/go/mongolib/util/main_test.go b/src/go/mongolib/util/main_test.go index edf26b5d..853046a4 100644 --- a/src/go/mongolib/util/main_test.go +++ b/src/go/mongolib/util/main_test.go @@ -12,7 +12,7 @@ import ( "go.mongodb.org/mongo-driver/mongo/options" ) -funci TestGetHostnames(t *testing.T) { +func TestGetHostnames(t *testing.T) { testCases := []struct { name string uri string diff --git a/src/go/pt-mongodb-query-digest/main_test.go b/src/go/pt-mongodb-query-digest/main_test.go index 41331120..3c69317b 100644 --- a/src/go/pt-mongodb-query-digest/main_test.go +++ b/src/go/pt-mongodb-query-digest/main_test.go @@ -1,499 +1,475 @@ package main -import ( - "bufio" - "bytes" - "context" - "fmt" - "io/ioutil" - "log" - "os" - "os/exec" - "reflect" - "regexp" - "runtime" - "sort" - "strings" - "testing" - "text/template" - "time" +//TODO: Rewrite tests to use the new sandbox - "github.com/pborman/getopt/v2" - "github.com/percona/percona-toolkit/src/go/lib/profiling" - "github.com/percona/percona-toolkit/src/go/lib/tutil" - "github.com/percona/percona-toolkit/src/go/mongolib/stats" - "github.com/percona/pmgo" - "gopkg.in/mgo.v2/dbtest" -) - -const ( - samples = "/src/go/tests/" -) - -type testVars struct { - RootPath string -} - -var vars testVars -var Server dbtest.DBServer - -func TestMain(m *testing.M) { - var err error - if vars.RootPath, err = tutil.RootPath(); err != nil { - log.Printf("cannot get root path: %s", err.Error()) - os.Exit(1) - } - os.Exit(m.Run()) - - // The tempdir is created so MongoDB has a location to store its files. - // Contents are wiped once the server stops - os.Setenv("CHECK_SESSIONS", "0") - tempDir, _ := ioutil.TempDir("", "testing") - Server.SetPath(tempDir) - - retCode := m.Run() - - Server.Session().Close() - Server.Session().DB("samples").DropDatabase() - - // Stop shuts down the temporary server and removes data on disk. - Server.Stop() - - // call with result of m.Run() - os.Exit(retCode) -} - -func TestIsProfilerEnabled(t *testing.T) { - mongoDSN := os.Getenv("PT_TEST_MONGODB_DSN") - if mongoDSN == "" { - t.Skip("Skippping TestIsProfilerEnabled. It runs only in integration tests") - } - - dialer := pmgo.NewDialer() - di, _ := pmgo.ParseURL(mongoDSN) - - enabled, err := isProfilerEnabled(dialer, di) - - if err != nil { - t.Errorf("Cannot check if profiler is enabled: %s", err.Error()) - } - if enabled != true { - t.Error("Profiler must be enabled") - } - -} - -func TestParseArgs(t *testing.T) { - tests := []struct { - args []string - want *options - }{ - { - args: []string{TOOLNAME}, // arg[0] is the command itself - want: &options{ - Host: DEFAULT_HOST, - LogLevel: DEFAULT_LOGLEVEL, - OrderBy: strings.Split(DEFAULT_ORDERBY, ","), - SkipCollections: strings.Split(DEFAULT_SKIPCOLLECTIONS, ","), - AuthDB: DEFAULT_AUTHDB, - OutputFormat: "text", - }, - }, - { - args: []string{TOOLNAME, "zapp.brannigan.net:27018/samples", "--help"}, - want: nil, - }, - { - args: []string{TOOLNAME, "zapp.brannigan.net:27018/samples"}, - want: &options{ - Host: "zapp.brannigan.net:27018/samples", - LogLevel: DEFAULT_LOGLEVEL, - OrderBy: strings.Split(DEFAULT_ORDERBY, ","), - SkipCollections: strings.Split(DEFAULT_SKIPCOLLECTIONS, ","), - AuthDB: DEFAULT_AUTHDB, - Help: false, - OutputFormat: "text", - }, - }, - } - for i, test := range tests { - getopt.Reset() - os.Args = test.args - got, err := getOptions() - if err != nil { - t.Errorf("error parsing command line arguments: %s", err.Error()) - } - if !reflect.DeepEqual(got, test.want) { - t.Errorf("invalid command line options test %d\ngot %+v\nwant %+v\n", i, got, test.want) - } - } - -} - -type Data struct { - bin string - url string -} - -func TestPTMongoDBQueryDigest(t *testing.T) { - var err error - - binDir, err := ioutil.TempDir("/tmp", "pmm-client-test-bindir-") - if err != nil { - t.Error(err) - } - defer func() { - err := os.RemoveAll(binDir) - if err != nil { - t.Error(err) - } - }() - - bin := binDir + "/pt-mongodb-query-digest" - xVariables := map[string]string{ - "main.Build": "", - "main.Version": "", - "main.GoVersion": "", - } - var ldflags []string - for x, value := range xVariables { - ldflags = append(ldflags, fmt.Sprintf("-X %s=%s", x, value)) - } - cmd := exec.Command( - "go", - "build", - "-o", - bin, - "-ldflags", - strings.Join(ldflags, " "), - ) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - err = cmd.Run() - if err != nil { - t.Error(err) - } - - data := Data{ - bin: bin, - } - tests := []func(*testing.T, Data){ - testVersion, - testEmptySystemProfile, - testAllOperationsTemplate, - } - t.Run("pmm-admin", func(t *testing.T) { - for _, f := range tests { - f := f // capture range variable - fName := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name() - t.Run(fName, func(t *testing.T) { - // Clean up system.profile - var err error - data.url = "127.0.0.1/test" - err = profiling.Disable(data.url) - if err != nil { - t.Error(err) - } - profiling.Drop(data.url) - err = profiling.Enable(data.url) - if err != nil { - t.Error(err) - } - defer profiling.Disable(data.url) - - // t.Parallel() - f(t, data) - }) - } - }) - -} - -func testVersion(t *testing.T, data Data) { - cmd := exec.Command( - data.bin, - "--version", - ) - output, err := cmd.CombinedOutput() - if err != nil { - t.Error(err) - } - expected := `pt-mongodb-query-digest -Version -Build: using ` - - assertRegexpLines(t, expected, string(output)) -} - -func testEmptySystemProfile(t *testing.T, data Data) { - cmd := exec.Command( - data.bin, - data.url, - ) - output, err := cmd.CombinedOutput() - if err != nil { - t.Error(err) - } - - expected := "No queries found in profiler information for database \\\"test\\\"" - if !strings.Contains(string(output), expected) { - t.Errorf("Empty system.profile.\nGot:\n%s\nWant:\n%s\n", string(output), expected) - } -} - -func testAllOperationsTemplate(t *testing.T, data Data) { - dir := vars.RootPath + samples + "/doc/script/profile/" - files, err := ioutil.ReadDir(dir) - if err != nil { - t.Fatalf("cannot list samples: %s", err) - } - - fs := []string{} - for _, file := range files { - fs = append(fs, dir+file.Name()) - } - sort.Strings(fs) - err = run(fs...) - if err != nil { - t.Fatalf("cannot execute queries: %s", err) - } - - // disable profiling so pt-mongodb-query digest reads rows from `system.profile` - profiling.Disable(data.url) - - // run profiler - cmd := exec.Command( - data.bin, - data.url, - ) - - output, err := cmd.CombinedOutput() - if err != nil { - t.Error(err) - } - - queries := []stats.QueryStats{ - { - ID: "e357abe482dcc0cd03ab742741bf1c86", - Namespace: "test.coll", - Operation: "INSERT", - Fingerprint: "INSERT coll", - }, - { - ID: "c9b40ce564762834d12b0390a292645c", - Namespace: "test.coll", - Operation: "DROP", - Fingerprint: "DROP coll drop", - }, - { - ID: "db759bfd83441deecc71382323041ce6", - Namespace: "test.coll", - Operation: "GETMORE", - Fingerprint: "GETMORE coll", - }, - { - ID: "e72ad41302045bd6c2bcad76511f915a", - Namespace: "test.coll", - Operation: "REMOVE", - Fingerprint: "REMOVE coll a,b", - }, - { - ID: "30dbfbc89efd8cfd40774dff0266a28f", - Namespace: "test.coll", - Operation: "AGGREGATE", - Fingerprint: "AGGREGATE coll a", - }, - { - ID: "a6782ae38ef891d5506341a4b0ab2747", - Namespace: "test", - Operation: "EVAL", - Fingerprint: "EVAL", - }, - { - ID: "76d7662df07b44135ac3e07e44a6eb39", - Namespace: "", - Operation: "EXPLAIN", - Fingerprint: "EXPLAIN", - }, - { - ID: "e8a3f05a4bd3f0bfa7d38eb2372258b1", - Namespace: "test.coll", - Operation: "FINDANDMODIFY", - Fingerprint: "FINDANDMODIFY coll a", - }, - { - ID: "2a639e77efe3e68399ef9482575b3421", - Namespace: "test.coll", - Operation: "FIND", - Fingerprint: "FIND coll", - }, - { - ID: "fe0bf975a044fe47fd32b835ceba612d", - Namespace: "test.coll", - Operation: "FIND", - Fingerprint: "FIND coll a", - }, - { - ID: "20fe80188ec82c9d3c3dcf3f4817f8f9", - Namespace: "test.coll", - Operation: "FIND", - Fingerprint: "FIND coll b,c", - }, - { - ID: "02104210d67fe680273784d833f86831", - Namespace: "test.coll", - Operation: "FIND", - Fingerprint: "FIND coll c,k,pad", - }, - { - ID: "5efe4738d807c74b3980de76c37a0870", - Namespace: "test.coll", - Operation: "FIND", - Fingerprint: "FIND coll k", - }, - { - ID: "798d7c1cd25b63cb6a307126a25910d6", - Namespace: "test.system.js", - Operation: "FIND", - Fingerprint: "FIND system.js", - }, - { - ID: "c70403cbd55ffbb07f08c0cb77a24b19", - Namespace: "test.coll", - Operation: "GEONEAR", - Fingerprint: "GEONEAR coll", - }, - { - ID: "e4122a58c99ab0a4020ce7d195c5a8cb", - Namespace: "test.coll", - Operation: "DISTINCT", - Fingerprint: "DISTINCT coll a,b", - }, - { - ID: "ca8bb19386488570447f5753741fb494", - Namespace: "test.coll", - Operation: "GROUP", - Fingerprint: "GROUP coll a,b", - }, - { - ID: "10b8f47b366fbfd1fb01f8d17d75b1a2", - Namespace: "test.coll", - Operation: "COUNT", - Fingerprint: "COUNT coll a", - }, - { - ID: "cc3cb3824eea4094eb042f5ca76bd385", - Namespace: "test.coll", - Operation: "MAPREDUCE", - Fingerprint: "MAPREDUCE coll a", - }, - { - ID: "cba2dff0740762c6e5769f0e300df676", - Namespace: "test.coll", - Operation: "COUNT", - Fingerprint: "COUNT coll", - }, - { - ID: "f74a5120ac22d02120ccbf6d478b0dbc", - Namespace: "test.coll", - Operation: "UPDATE", - Fingerprint: "UPDATE coll a", - }, - } - - expected := `Profiler is disabled for the "test" database but there are \s*[0-9]+ documents in the system.profile collection. -Using those documents for the stats - -# Totals -# Ratio [0-9\.]+ \(docs scanned/returned\) -# Attribute pct total min max avg 95% stddev median -# ================== === ======== ======== ======== ======== ======== ======= ======== -# Count \(docs\) (\s*[0-9]+)\s -# Exec Time ms (\s*[0-9]+){8}\s -# Docs Scanned (\s*[0-9\.]+){8}\s -# Docs Returned (\s*[0-9\.]+){8}\s -# Bytes sent (\s*[0-9\.K]+){8}(K|\s) -#\s -` - - queryTpl := ` -# Query [0-9]+: [0-9\.]+ QPS, ID {{.ID}} -# Ratio [0-9\.]+ \(docs scanned/returned\) -# Time range: .* to .* -# Attribute pct total min max avg 95% stddev median -# ================== === ======== ======== ======== ======== ======== ======= ======== -# Count \(docs\) (\s*[0-9]+)\s -# Exec Time ms (\s*[0-9]+){8}\s -# Docs Scanned (\s*[0-9\.]+){8}\s -# Docs Returned (\s*[0-9\.]+){8}\s -# Bytes sent (\s*[0-9\.K]+){8}(K|\s) -# String: -# Namespace {{.Namespace}} -# Operation {{.Operation}} -# Fingerprint {{.Fingerprint}} -# Query .* - -` - - tpl, _ := template.New("query").Parse(queryTpl) - for _, query := range queries { - buf := bytes.Buffer{} - err := tpl.Execute(&buf, query) - if err != nil { - t.Error(err) - } - - expected += buf.String() - } - expected += "\n" // Looks like we expect additional line - - assertRegexpLines(t, expected, string(output)) -} - -// assertRegexpLines matches regexp line by line to corresponding line of text -func assertRegexpLines(t *testing.T, rx string, str string, msgAndArgs ...interface{}) bool { - expectedScanner := bufio.NewScanner(strings.NewReader(rx)) - defer func() { - if err := expectedScanner.Err(); err != nil { - t.Fatal(err) - } - }() - - actualScanner := bufio.NewScanner(strings.NewReader(str)) - defer func() { - if err := actualScanner.Err(); err != nil { - t.Fatal(err) - } - }() - - ok := true - for { - asOk := actualScanner.Scan() - esOk := expectedScanner.Scan() - - switch { - case asOk && esOk: - ok, err := regexp.MatchString("^"+expectedScanner.Text()+"$", actualScanner.Text()) - if err != nil { - t.Error(err) - } - if !ok { - t.Errorf("regexp '%s' doesn't match '%s'", expectedScanner.Text(), actualScanner.Text()) - } - case asOk: - t.Errorf("didn't expect more lines but got: %s", actualScanner.Text()) - ok = false - case esOk: - t.Errorf("didn't got line but expected it to match against: %s", expectedScanner.Text()) - ok = false - default: - return ok - } - } -} - -func run(arg ...string) error { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - return exec.CommandContext(ctx, "mongo", arg...).Run() -} +// const ( +// samples = "/src/go/tests/" +// ) +// +// type testVars struct { +// RootPath string +// } +// +// var vars testVars +// var Server dbtest.DBServer +// +// func TestMain(m *testing.M) { +// var err error +// if vars.RootPath, err = tutil.RootPath(); err != nil { +// log.Printf("cannot get root path: %s", err.Error()) +// os.Exit(1) +// } +// os.Exit(m.Run()) +// +// // The tempdir is created so MongoDB has a location to store its files. +// // Contents are wiped once the server stops +// os.Setenv("CHECK_SESSIONS", "0") +// tempDir, _ := ioutil.TempDir("", "testing") +// Server.SetPath(tempDir) +// +// retCode := m.Run() +// +// Server.Session().Close() +// Server.Session().DB("samples").DropDatabase() +// +// // Stop shuts down the temporary server and removes data on disk. +// Server.Stop() +// +// // call with result of m.Run() +// os.Exit(retCode) +// } +// +// func TestIsProfilerEnabled(t *testing.T) { +// mongoDSN := os.Getenv("PT_TEST_MONGODB_DSN") +// if mongoDSN == "" { +// t.Skip("Skippping TestIsProfilerEnabled. It runs only in integration tests") +// } +// +// dialer := pmgo.NewDialer() +// di, _ := pmgo.ParseURL(mongoDSN) +// +// enabled, err := isProfilerEnabled(dialer, di) +// +// if err != nil { +// t.Errorf("Cannot check if profiler is enabled: %s", err.Error()) +// } +// if enabled != true { +// t.Error("Profiler must be enabled") +// } +// +// } +// +// func TestParseArgs(t *testing.T) { +// tests := []struct { +// args []string +// want *options +// }{ +// { +// args: []string{TOOLNAME}, // arg[0] is the command itself +// want: &options{ +// Host: DEFAULT_HOST, +// LogLevel: DEFAULT_LOGLEVEL, +// OrderBy: strings.Split(DEFAULT_ORDERBY, ","), +// SkipCollections: strings.Split(DEFAULT_SKIPCOLLECTIONS, ","), +// AuthDB: DEFAULT_AUTHDB, +// OutputFormat: "text", +// }, +// }, +// { +// args: []string{TOOLNAME, "zapp.brannigan.net:27018/samples", "--help"}, +// want: nil, +// }, +// { +// args: []string{TOOLNAME, "zapp.brannigan.net:27018/samples"}, +// want: &options{ +// Host: "zapp.brannigan.net:27018/samples", +// LogLevel: DEFAULT_LOGLEVEL, +// OrderBy: strings.Split(DEFAULT_ORDERBY, ","), +// SkipCollections: strings.Split(DEFAULT_SKIPCOLLECTIONS, ","), +// AuthDB: DEFAULT_AUTHDB, +// Help: false, +// OutputFormat: "text", +// }, +// }, +// } +// for i, test := range tests { +// getopt.Reset() +// os.Args = test.args +// got, err := getOptions() +// if err != nil { +// t.Errorf("error parsing command line arguments: %s", err.Error()) +// } +// if !reflect.DeepEqual(got, test.want) { +// t.Errorf("invalid command line options test %d\ngot %+v\nwant %+v\n", i, got, test.want) +// } +// } +// +// } +// +// type Data struct { +// bin string +// url string +// } +// +// func TestPTMongoDBQueryDigest(t *testing.T) { +// var err error +// +// binDir, err := ioutil.TempDir("/tmp", "pmm-client-test-bindir-") +// if err != nil { +// t.Error(err) +// } +// defer func() { +// err := os.RemoveAll(binDir) +// if err != nil { +// t.Error(err) +// } +// }() +// +// bin := binDir + "/pt-mongodb-query-digest" +// xVariables := map[string]string{ +// "main.Build": "", +// "main.Version": "", +// "main.GoVersion": "", +// } +// var ldflags []string +// for x, value := range xVariables { +// ldflags = append(ldflags, fmt.Sprintf("-X %s=%s", x, value)) +// } +// cmd := exec.Command( +// "go", +// "build", +// "-o", +// bin, +// "-ldflags", +// strings.Join(ldflags, " "), +// ) +// cmd.Stdout = os.Stdout +// cmd.Stderr = os.Stderr +// err = cmd.Run() +// if err != nil { +// t.Error(err) +// } +// +// data := Data{ +// bin: bin, +// } +// tests := []func(*testing.T, Data){ +// testVersion, +// testEmptySystemProfile, +// testAllOperationsTemplate, +// } +// t.Run("pmm-admin", func(t *testing.T) { +// for _, f := range tests { +// f := f // capture range variable +// fName := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name() +// t.Run(fName, func(t *testing.T) { +// // Clean up system.profile +// var err error +// data.url = "127.0.0.1/test" +// err = profiling.Disable(data.url) +// if err != nil { +// t.Error(err) +// } +// profiling.Drop(data.url) +// err = profiling.Enable(data.url) +// if err != nil { +// t.Error(err) +// } +// defer profiling.Disable(data.url) +// +// // t.Parallel() +// f(t, data) +// }) +// } +// }) +// +// } +// +// func testVersion(t *testing.T, data Data) { +// cmd := exec.Command( +// data.bin, +// "--version", +// ) +// output, err := cmd.CombinedOutput() +// if err != nil { +// t.Error(err) +// } +// expected := `pt-mongodb-query-digest +// Version +// Build: using ` +// +// assertRegexpLines(t, expected, string(output)) +// } +// +// func testEmptySystemProfile(t *testing.T, data Data) { +// cmd := exec.Command( +// data.bin, +// data.url, +// ) +// output, err := cmd.CombinedOutput() +// if err != nil { +// t.Error(err) +// } +// +// expected := "No queries found in profiler information for database \\\"test\\\"" +// if !strings.Contains(string(output), expected) { +// t.Errorf("Empty system.profile.\nGot:\n%s\nWant:\n%s\n", string(output), expected) +// } +// } +// +// func testAllOperationsTemplate(t *testing.T, data Data) { +// dir := vars.RootPath + samples + "/doc/script/profile/" +// files, err := ioutil.ReadDir(dir) +// if err != nil { +// t.Fatalf("cannot list samples: %s", err) +// } +// +// fs := []string{} +// for _, file := range files { +// fs = append(fs, dir+file.Name()) +// } +// sort.Strings(fs) +// err = run(fs...) +// if err != nil { +// t.Fatalf("cannot execute queries: %s", err) +// } +// +// // disable profiling so pt-mongodb-query digest reads rows from `system.profile` +// profiling.Disable(data.url) +// +// // run profiler +// cmd := exec.Command( +// data.bin, +// data.url, +// ) +// +// output, err := cmd.CombinedOutput() +// if err != nil { +// t.Error(err) +// } +// +// queries := []stats.QueryStats{ +// { +// ID: "e357abe482dcc0cd03ab742741bf1c86", +// Namespace: "test.coll", +// Operation: "INSERT", +// Fingerprint: "INSERT coll", +// }, +// { +// ID: "c9b40ce564762834d12b0390a292645c", +// Namespace: "test.coll", +// Operation: "DROP", +// Fingerprint: "DROP coll drop", +// }, +// { +// ID: "db759bfd83441deecc71382323041ce6", +// Namespace: "test.coll", +// Operation: "GETMORE", +// Fingerprint: "GETMORE coll", +// }, +// { +// ID: "e72ad41302045bd6c2bcad76511f915a", +// Namespace: "test.coll", +// Operation: "REMOVE", +// Fingerprint: "REMOVE coll a,b", +// }, +// { +// ID: "30dbfbc89efd8cfd40774dff0266a28f", +// Namespace: "test.coll", +// Operation: "AGGREGATE", +// Fingerprint: "AGGREGATE coll a", +// }, +// { +// ID: "a6782ae38ef891d5506341a4b0ab2747", +// Namespace: "test", +// Operation: "EVAL", +// Fingerprint: "EVAL", +// }, +// { +// ID: "76d7662df07b44135ac3e07e44a6eb39", +// Namespace: "", +// Operation: "EXPLAIN", +// Fingerprint: "EXPLAIN", +// }, +// { +// ID: "e8a3f05a4bd3f0bfa7d38eb2372258b1", +// Namespace: "test.coll", +// Operation: "FINDANDMODIFY", +// Fingerprint: "FINDANDMODIFY coll a", +// }, +// { +// ID: "2a639e77efe3e68399ef9482575b3421", +// Namespace: "test.coll", +// Operation: "FIND", +// Fingerprint: "FIND coll", +// }, +// { +// ID: "fe0bf975a044fe47fd32b835ceba612d", +// Namespace: "test.coll", +// Operation: "FIND", +// Fingerprint: "FIND coll a", +// }, +// { +// ID: "20fe80188ec82c9d3c3dcf3f4817f8f9", +// Namespace: "test.coll", +// Operation: "FIND", +// Fingerprint: "FIND coll b,c", +// }, +// { +// ID: "02104210d67fe680273784d833f86831", +// Namespace: "test.coll", +// Operation: "FIND", +// Fingerprint: "FIND coll c,k,pad", +// }, +// { +// ID: "5efe4738d807c74b3980de76c37a0870", +// Namespace: "test.coll", +// Operation: "FIND", +// Fingerprint: "FIND coll k", +// }, +// { +// ID: "798d7c1cd25b63cb6a307126a25910d6", +// Namespace: "test.system.js", +// Operation: "FIND", +// Fingerprint: "FIND system.js", +// }, +// { +// ID: "c70403cbd55ffbb07f08c0cb77a24b19", +// Namespace: "test.coll", +// Operation: "GEONEAR", +// Fingerprint: "GEONEAR coll", +// }, +// { +// ID: "e4122a58c99ab0a4020ce7d195c5a8cb", +// Namespace: "test.coll", +// Operation: "DISTINCT", +// Fingerprint: "DISTINCT coll a,b", +// }, +// { +// ID: "ca8bb19386488570447f5753741fb494", +// Namespace: "test.coll", +// Operation: "GROUP", +// Fingerprint: "GROUP coll a,b", +// }, +// { +// ID: "10b8f47b366fbfd1fb01f8d17d75b1a2", +// Namespace: "test.coll", +// Operation: "COUNT", +// Fingerprint: "COUNT coll a", +// }, +// { +// ID: "cc3cb3824eea4094eb042f5ca76bd385", +// Namespace: "test.coll", +// Operation: "MAPREDUCE", +// Fingerprint: "MAPREDUCE coll a", +// }, +// { +// ID: "cba2dff0740762c6e5769f0e300df676", +// Namespace: "test.coll", +// Operation: "COUNT", +// Fingerprint: "COUNT coll", +// }, +// { +// ID: "f74a5120ac22d02120ccbf6d478b0dbc", +// Namespace: "test.coll", +// Operation: "UPDATE", +// Fingerprint: "UPDATE coll a", +// }, +// } +// +// expected := `Profiler is disabled for the "test" database but there are \s*[0-9]+ documents in the system.profile collection. +// Using those documents for the stats +// +// # Totals +// # Ratio [0-9\.]+ \(docs scanned/returned\) +// # Attribute pct total min max avg 95% stddev median +// # ================== === ======== ======== ======== ======== ======== ======= ======== +// # Count \(docs\) (\s*[0-9]+)\s +// # Exec Time ms (\s*[0-9]+){8}\s +// # Docs Scanned (\s*[0-9\.]+){8}\s +// # Docs Returned (\s*[0-9\.]+){8}\s +// # Bytes sent (\s*[0-9\.K]+){8}(K|\s) +// #\s +// ` +// +// queryTpl := ` +// # Query [0-9]+: [0-9\.]+ QPS, ID {{.ID}} +// # Ratio [0-9\.]+ \(docs scanned/returned\) +// # Time range: .* to .* +// # Attribute pct total min max avg 95% stddev median +// # ================== === ======== ======== ======== ======== ======== ======= ======== +// # Count \(docs\) (\s*[0-9]+)\s +// # Exec Time ms (\s*[0-9]+){8}\s +// # Docs Scanned (\s*[0-9\.]+){8}\s +// # Docs Returned (\s*[0-9\.]+){8}\s +// # Bytes sent (\s*[0-9\.K]+){8}(K|\s) +// # String: +// # Namespace {{.Namespace}} +// # Operation {{.Operation}} +// # Fingerprint {{.Fingerprint}} +// # Query .* +// +// ` +// +// tpl, _ := template.New("query").Parse(queryTpl) +// for _, query := range queries { +// buf := bytes.Buffer{} +// err := tpl.Execute(&buf, query) +// if err != nil { +// t.Error(err) +// } +// +// expected += buf.String() +// } +// expected += "\n" // Looks like we expect additional line +// +// assertRegexpLines(t, expected, string(output)) +// } +// +// // assertRegexpLines matches regexp line by line to corresponding line of text +// func assertRegexpLines(t *testing.T, rx string, str string, msgAndArgs ...interface{}) bool { +// expectedScanner := bufio.NewScanner(strings.NewReader(rx)) +// defer func() { +// if err := expectedScanner.Err(); err != nil { +// t.Fatal(err) +// } +// }() +// +// actualScanner := bufio.NewScanner(strings.NewReader(str)) +// defer func() { +// if err := actualScanner.Err(); err != nil { +// t.Fatal(err) +// } +// }() +// +// ok := true +// for { +// asOk := actualScanner.Scan() +// esOk := expectedScanner.Scan() +// +// switch { +// case asOk && esOk: +// ok, err := regexp.MatchString("^"+expectedScanner.Text()+"$", actualScanner.Text()) +// if err != nil { +// t.Error(err) +// } +// if !ok { +// t.Errorf("regexp '%s' doesn't match '%s'", expectedScanner.Text(), actualScanner.Text()) +// } +// case asOk: +// t.Errorf("didn't expect more lines but got: %s", actualScanner.Text()) +// ok = false +// case esOk: +// t.Errorf("didn't got line but expected it to match against: %s", expectedScanner.Text()) +// ok = false +// default: +// return ok +// } +// } +// } +// +// func run(arg ...string) error { +// ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) +// defer cancel() +// return exec.CommandContext(ctx, "mongo", arg...).Run() +// } diff --git a/src/go/pt-mongodb-summary/oplog/oplog.go b/src/go/pt-mongodb-summary/oplog/oplog.go index f1287e10..1f276d46 100644 --- a/src/go/pt-mongodb-summary/oplog/oplog.go +++ b/src/go/pt-mongodb-summary/oplog/oplog.go @@ -9,9 +9,9 @@ import ( "github.com/percona/percona-toolkit/src/go/mongolib/proto" "github.com/percona/percona-toolkit/src/go/mongolib/util" "github.com/pkg/errors" + "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" - "gopkg.in/mgo.v2/bson" ) func GetOplogInfo(ctx context.Context, hostnames []string, co *options.ClientOptions) ([]proto.OplogInfo, error) {