Fixed explain tests

Updated travis.yml to support replicasets
This commit is contained in:
Carlos Salguero
2019-08-09 15:53:40 -03:00
parent 7466e9b1ba
commit e13368725f
7 changed files with 317 additions and 61 deletions

View File

@@ -9,6 +9,34 @@ services:
- docker
env:
DOCKER_COMPOSE_VERSION: 1.8.0
TEST_MONGODB_FLAVOR: mongo
TEST_PSMDB_VERSION: 4.0
TEST_MONGODB_ADMIN_USERNAME: admin
TEST_MONGODB_ADMIN_PASSWORD: admin123456
TEST_MONGODB_USERNAME: test
TEST_MONGODB_PASSWORD: 123456
TEST_MONGODB_STANDALONE_PORT: 27017
TEST_MONGODB_MONGOS_PORT: 17000
TEST_MONGODB_S1_RS: rs1
TEST_MONGODB_S1_PRIMARY_PORT: 17001
TEST_MONGODB_S1_SECONDARY1_PORT: 17002
TEST_MONGODB_S1_SECONDARY2_PORT: 17003
TEST_MONGODB_S2_RS: rs2
TEST_MONGODB_S2_PRIMARY_PORT: 17004
TEST_MONGODB_S2_SECONDARY1_PORT: 17005
TEST_MONGODB_S2_SECONDARY2_PORT: 17006
TEST_MONGODB_CONFIGSVR_RS: csReplSet
TEST_MONGODB_CONFIGSVR1_PORT: 17007
TEST_MONGODB_CONFIGSVR2_PORT: 17008
TEST_MONGODB_CONFIGSVR3_PORT: 17009
TEST_MONGODB_S3_RS: rs3
TEST_MONGODB_S3_PRIMARY_PORT: 17021
TEST_MONGODB_S3_SECONDARY1_PORT: 17022
TEST_MONGODB_S3_SECONDARY2_PORT: 17023
MINIO_ENDPOINT: http://localhost:9000/
MINIO_ACCESS_KEY_ID: example00000
MINIO_SECRET_ACCESS_KEY: secret00000
matrix:
- MONGODB_IMAGE=mongo:3.0
- MONGODB_IMAGE=mongo:3.2
@@ -17,6 +45,11 @@ env:
- MONGODB_IMAGE=percona/percona-server-mongodb:3.2
- MONGODB_IMAGE=percona/percona-server-mongodb:3.4
before_install:
- sudo apt-get update
- sudo apt-get install -o Dpkg::Options::="--force-confold" --force-yes -y docker-engine
- docker-compose --version
install:
- go get -u github.com/golang/dep/cmd/dep

View File

@@ -1,6 +1,204 @@
# version: '3'
# services:
# mongo:
# image: ${MONGODB_IMAGE:-percona/percona-server-mongodb:3.4}
# ports:
# - "27017:27017"
---
version: '3'
services:
mongo:
image: ${MONGODB_IMAGE:-percona/percona-server-mongodb:3.4}
ports:
- "27017:27017"
standalone:
network_mode: host
image: ${TEST_MONGODB_FLAVOR}:${TEST_PSMDB_VERSION}
command: --port=27017
volumes:
- src/go/docker/test/entrypoint-mongod.sh:/entrypoint.sh:ro
- src/go/docker/test/entrypoint-mongod.sh:/usr/local/bin/docker-entrypoint.sh:ro
- src/go/docker/test/mongod.key:/mongod.key:ro
- src/go/docker/test/ssl/rootCA.crt:/rootCA.crt:ro
- src/go/docker/test/ssl/mongodb.pem:/mongod.pem:ro
s1-mongo1:
network_mode: host
image: ${TEST_MONGODB_FLAVOR}:${TEST_PSMDB_VERSION}
command: --replSet=${TEST_MONGODB_S1_RS} --port=${TEST_MONGODB_S1_PRIMARY_PORT} --shardsvr
volumes:
- src/go/docker/test/entrypoint-mongod.sh:/entrypoint.sh:ro
- src/go/docker/test/entrypoint-mongod.sh:/usr/local/bin/docker-entrypoint.sh:ro
- src/go/docker/test/mongod.key:/mongod.key:ro
- src/go/docker/test/ssl/rootCA.crt:/rootCA.crt:ro
- src/go/docker/test/ssl/mongodb.pem:/mongod.pem:ro
s1-mongo2:
network_mode: host
image: ${TEST_MONGODB_FLAVOR}:${TEST_PSMDB_VERSION}
command: --replSet=${TEST_MONGODB_S1_RS} --port=${TEST_MONGODB_S1_SECONDARY1_PORT} --shardsvr
volumes:
- src/go/docker/test/entrypoint-mongod.sh:/entrypoint.sh:ro
- src/go/docker/test/entrypoint-mongod.sh:/usr/local/bin/docker-entrypoint.sh:ro
- src/go/docker/test/mongod.key:/mongod.key:ro
- src/go/docker/test/ssl/rootCA.crt:/rootCA.crt:ro
- src/go/docker/test/ssl/mongodb.pem:/mongod.pem:ro
s1-mongo3:
network_mode: host
image: ${TEST_MONGODB_FLAVOR}:${TEST_PSMDB_VERSION}
command: --replSet=${TEST_MONGODB_S1_RS} --port=${TEST_MONGODB_S1_SECONDARY2_PORT} --shardsvr
volumes:
- src/go/docker/test/entrypoint-mongod.sh:/entrypoint.sh:ro
- src/go/docker/test/entrypoint-mongod.sh:/usr/local/bin/docker-entrypoint.sh:ro
- src/go/docker/test/mongod.key:/mongod.key:ro
- src/go/docker/test/ssl/rootCA.crt:/rootCA.crt:ro
- src/go/docker/test/ssl/mongodb.pem:/mongod.pem:ro
s2-mongo1:
network_mode: host
image: ${TEST_MONGODB_FLAVOR}:${TEST_PSMDB_VERSION}
command: --replSet=${TEST_MONGODB_S2_RS} --port=${TEST_MONGODB_S2_PRIMARY_PORT} --shardsvr
volumes:
- src/go/docker/test/entrypoint-mongod.sh:/entrypoint.sh:ro
- src/go/docker/test/entrypoint-mongod.sh:/usr/local/bin/docker-entrypoint.sh:ro
- src/go/docker/test/mongod.key:/mongod.key:ro
- src/go/docker/test/ssl/rootCA.crt:/rootCA.crt:ro
- src/go/docker/test/ssl/mongodb.pem:/mongod.pem:ro
s2-mongo2:
network_mode: host
image: ${TEST_MONGODB_FLAVOR}:${TEST_PSMDB_VERSION}
command: --replSet=${TEST_MONGODB_S2_RS} --port=${TEST_MONGODB_S2_SECONDARY1_PORT} --shardsvr
volumes:
- src/go/docker/test/entrypoint-mongod.sh:/entrypoint.sh:ro
- src/go/docker/test/entrypoint-mongod.sh:/usr/local/bin/docker-entrypoint.sh:ro
- src/go/docker/test/mongod.key:/mongod.key:ro
- src/go/docker/test/ssl/rootCA.crt:/rootCA.crt:ro
- src/go/docker/test/ssl/mongodb.pem:/mongod.pem:ro
s2-mongo3:
network_mode: host
image: ${TEST_MONGODB_FLAVOR}:${TEST_PSMDB_VERSION}
command: --replSet=${TEST_MONGODB_S2_RS} --port=${TEST_MONGODB_S2_SECONDARY2_PORT} --shardsvr
volumes:
- src/go/docker/test/entrypoint-mongod.sh:/entrypoint.sh:ro
- src/go/docker/test/entrypoint-mongod.sh:/usr/local/bin/docker-entrypoint.sh:ro
- src/go/docker/test/mongod.key:/mongod.key:ro
- src/go/docker/test/ssl/rootCA.crt:/rootCA.crt:ro
- src/go/docker/test/ssl/mongodb.pem:/mongod.pem:ro
s3-mongo1:
network_mode: host
image: ${TEST_MONGODB_FLAVOR}:${TEST_PSMDB_VERSION}
command: --replSet=${TEST_MONGODB_S3_RS} --port=${TEST_MONGODB_S3_PRIMARY_PORT}
volumes:
- src/go/docker/test/entrypoint-mongod.sh:/entrypoint.sh:ro
- src/go/docker/test/entrypoint-mongod.sh:/usr/local/bin/docker-entrypoint.sh:ro
- src/go/docker/test/mongod.key:/mongod.key:ro
- src/go/docker/test/ssl/rootCA.crt:/rootCA.crt:ro
- src/go/docker/test/ssl/mongodb.pem:/mongod.pem:ro
s3-mongo2:
network_mode: host
image: ${TEST_MONGODB_FLAVOR}:${TEST_PSMDB_VERSION}
command: --replSet=${TEST_MONGODB_S3_RS} --port=${TEST_MONGODB_S3_SECONDARY1_PORT}
volumes:
- src/go/docker/test/entrypoint-mongod.sh:/entrypoint.sh:ro
- src/go/docker/test/entrypoint-mongod.sh:/usr/local/bin/docker-entrypoint.sh:ro
- src/go/docker/test/mongod.key:/mongod.key:ro
- src/go/docker/test/ssl/rootCA.crt:/rootCA.crt:ro
- src/go/docker/test/ssl/mongodb.pem:/mongod.pem:ro
s3-mongo3:
network_mode: host
image: ${TEST_MONGODB_FLAVOR}:${TEST_PSMDB_VERSION}
command: --replSet=${TEST_MONGODB_S3_RS} --port=${TEST_MONGODB_S3_SECONDARY2_PORT}
volumes:
- src/go/docker/test/entrypoint-mongod.sh:/entrypoint.sh:ro
- src/go/docker/test/entrypoint-mongod.sh:/usr/local/bin/docker-entrypoint.sh:ro
- src/go/docker/test/mongod.key:/mongod.key:ro
- src/go/docker/test/ssl/rootCA.crt:/rootCA.crt:ro
- src/go/docker/test/ssl/mongodb.pem:/mongod.pem:ro
configsvr1:
network_mode: host
image: ${TEST_MONGODB_FLAVOR}:${TEST_PSMDB_VERSION}
command: --replSet=${TEST_MONGODB_CONFIGSVR_RS} --port=${TEST_MONGODB_CONFIGSVR1_PORT} --configsvr
volumes:
- src/go/docker/test/entrypoint-mongod.sh:/entrypoint.sh:ro
- src/go/docker/test/entrypoint-mongod.sh:/usr/local/bin/docker-entrypoint.sh:ro
- src/go/docker/test/mongod.key:/mongod.key:ro
- src/go/docker/test/ssl/rootCA.crt:/rootCA.crt:ro
- src/go/docker/test/ssl/mongodb.pem:/mongod.pem:ro
# configsvr2:
# network_mode: host
# image: ${TEST_MONGODB_FLAVOR}:${TEST_PSMDB_VERSION}
# command: --replSet=${TEST_MONGODB_CONFIGSVR_RS} --port=${TEST_MONGODB_CONFIGSVR2_PORT} --configsvr
# volumes:
# - src/go/docker/test/entrypoint-mongod.sh:/entrypoint.sh:ro
# - src/go/docker/test/mongod.key:/mongod.key:ro
# - src/go/docker/test/ssl/rootCA.crt:/rootCA.crt:ro
# - src/go/docker/test/ssl/mongodb.pem:/mongod.pem:ro
# configsvr3:
# network_mode: host
# image: ${TEST_MONGODB_FLAVOR}:${TEST_PSMDB_VERSION}
# command: --replSet=${TEST_MONGODB_CONFIGSVR_RS} --port=${TEST_MONGODB_CONFIGSVR3_PORT} --configsvr
# volumes:
# - src/go/docker/test/entrypoint-mongod.sh:/entrypoint.sh:ro
# - src/go/docker/test/mongod.key:/mongod.key:ro
# - src/go/docker/test/ssl/rootCA.crt:/rootCA.crt:ro
# - src/go/docker/test/ssl/mongodb.pem:/mongod.pem:ro
mongos:
network_mode: host
image: ${TEST_MONGODB_FLAVOR}:${TEST_PSMDB_VERSION}
command: --port=${TEST_MONGODB_MONGOS_PORT} --configdb=${TEST_MONGODB_CONFIGSVR_RS}/127.0.0.1:${TEST_MONGODB_CONFIGSVR1_PORT}
volumes:
- src/go/docker/test/entrypoint-mongos.sh:/entrypoint.sh:ro
- src/go/docker/test/entrypoint-mongos.sh:/usr/local/bin/docker-entrypoint.sh:ro
- src/go/docker/test/mongod.key:/mongos.key:ro
- src/go/docker/test/ssl/rootCA.crt:/rootCA.crt:ro
- src/go/docker/test/ssl/mongodb.pem:/mongos.pem:ro
depends_on:
- configsvr1
# - configsvr2
# - configsvr3
minio:
network_mode: host
image: minio/minio
environment:
- MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY_ID}
- MINIO_SECRET_KEY=${MINIO_SECRET_ACCESS_KEY}
command: server /data
init:
network_mode: host
image: ${TEST_MONGODB_FLAVOR}:${TEST_PSMDB_VERSION}
volumes:
- src/go/docker/test/init-cluster.sh:/entrypoint.sh:ro
- src/go/docker/test/init-cluster.sh:/usr/local/bin/docker-entrypoint.sh:ro
- src/go/docker/test/mongod.key:/mongod.key:ro
- src/go/docker/test/ssl/rootCA.crt:/rootCA.crt:ro
- src/go/docker/test/ssl/client.pem:/client.pem:ro
env_file:
- .env
depends_on:
- mongos
- s1-mongo1
- s1-mongo2
- s1-mongo3
- s2-mongo1
- s2-mongo2
- s2-mongo3
- s3-mongo1
- s3-mongo2
- s3-mongo3
- standalone
- minio
test:
build:
dockerfile: docker/test/Dockerfile
context: .
args:
- GOLANG_DOCKERHUB_TAG=${GOLANG_DOCKERHUB_TAG}
network_mode: host
env_file:
- .env
volumes:
- ./test-out:/tmp/out
- src/go/docker/test/ssl/rootCA.crt:/rootCA.crt:ro
- src/go/docker/test/ssl/client.pem:/client.pem:ro
depends_on:
- mongos
- s1-mongo1
- s1-mongo2
- s1-mongo3
- s2-mongo1
- s2-mongo2
- s2-mongo3
- init

View File

@@ -1,5 +1,5 @@
AWS_ACCESS_KEY_ID=AKIARXP3OARBNV35P5M5
AWS_SECRET_ACCESS_KEY=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
AWS_ACCESS_KEY_ID=AKIAJQ2GZPAJ3JZS52HQ
AWS_SECRET_ACCESS_KEY=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
GOCACHE=
GOLANG_DOCKERHUB_TAG=1.10-stretch
TEST_MONGODB_ADMIN_USERNAME=admin
@@ -24,5 +24,5 @@ TEST_MONGODB_CONFIGSVR1_PORT=17007
TEST_MONGODB_CONFIGSVR2_PORT=17008
TEST_MONGODB_CONFIGSVR3_PORT=17009
TEST_MONGODB_MONGOS_PORT=17000
TEST_PSMDB_VERSION=4.0
TEST_MONGODB_FLAVOR=mongo
TEST_PSMDB_VERSION=3.6
TEST_MONGODB_FLAVOR=percona/percona-server-mongodb

View File

@@ -7,8 +7,6 @@ import (
"os/exec"
"regexp"
"strings"
"go.mongodb.org/mongo-driver/bson"
)
const (
@@ -67,6 +65,8 @@ func LoadBson(filename string, destination interface{}) error {
re = regexp.MustCompile(`NumberLong\((.*)\)`)
buf = re.ReplaceAll(buf, []byte(`$1`))
re = regexp.MustCompile(`ISODate\((.*)\)`)
buf = re.ReplaceAll(buf, []byte(`$1`))
// Using regexp is not supported
// https://github.com/go-mgo/mgo/issues/363
re = regexp.MustCompile(`(/.*/)`)
@@ -77,7 +77,7 @@ func LoadBson(filename string, destination interface{}) error {
re = regexp.MustCompile(`(?s): (function \(.*?\) {.*?})`)
buf = re.ReplaceAll(buf, []byte(`: ""`))
err = bson.Unmarshal(buf, &destination)
err = json.Unmarshal(buf, &destination)
if err != nil {
return err
}

View File

@@ -1,21 +1,24 @@
package explain
import (
"context"
"fmt"
"github.com/percona/pmgo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"github.com/percona/percona-toolkit/src/go/mongolib/proto"
)
type explain struct {
session pmgo.SessionManager
ctx context.Context
client *mongo.Client
}
func New(session pmgo.SessionManager) *explain {
func New(ctx context.Context, client *mongo.Client) *explain {
return &explain{
session: session,
ctx: ctx,
client: client,
}
}
@@ -33,8 +36,12 @@ func (e *explain) Explain(db string, query []byte) ([]byte, error) {
}
var result proto.BsonD
err = e.session.DB(db).Run(eq.ExplainCmd(), &result)
if err != nil {
res := e.client.Database(db).RunCommand(e.ctx, eq.ExplainCmd())
if res.Err() != nil {
return nil, res.Err()
}
if err := res.Decode(&result); err != nil {
return nil, err
}

View File

@@ -1,17 +1,22 @@
package explain
import (
"context"
"fmt"
"io/ioutil"
"log"
"os"
"strings"
"testing"
"time"
"github.com/Masterminds/semver"
"github.com/percona/pmgo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
tu "github.com/percona/percona-toolkit/src/go/internal/testutils"
"github.com/percona/percona-toolkit/src/go/lib/tutil"
"github.com/percona/percona-toolkit/src/go/mongolib/proto"
)
@@ -38,17 +43,17 @@ func TestMain(m *testing.M) {
func TestExplain(t *testing.T) {
t.Parallel()
dialer := pmgo.NewDialer()
dialInfo, err := pmgo.ParseURL("")
uri := fmt.Sprintf("mongodb://%s:%s@%s:%s", tu.MongoDBUser, tu.MongoDBPassword, tu.MongoDBHost, tu.MongoDBMongosPort)
client, err := mongo.NewClient(options.Client().ApplyURI(uri))
if err != nil {
t.Fatalf("cannot parse URL: %s", err)
t.Fatalf("cannot get a new MongoDB client: %s", err)
}
session, err := dialer.DialWithInfo(dialInfo)
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
err = client.Connect(ctx)
if err != nil {
t.Fatalf("cannot dial to MongoDB: %s", err)
t.Fatalf("Cannot connect to MongoDB: %s", err)
}
defer session.Close()
dir := vars.RootPath + samples + "/doc/out/"
files, err := ioutil.ReadDir(dir)
@@ -56,9 +61,13 @@ func TestExplain(t *testing.T) {
t.Fatalf("cannot list samples: %s", err)
}
bi, err := session.BuildInfo()
if err != nil {
t.Fatalf("cannot get BuildInfo: %s", err)
res := client.Database("admin").RunCommand(ctx, primitive.M{"buildInfo": 1})
if res.Err() != nil {
t.Fatalf("Cannot get buildInfo: %s", err)
}
bi := proto.BuildInfo{}
if err := res.Decode(&bi); err != nil {
t.Fatalf("Cannot decode buildInfo response: %s", err)
}
versions := []string{
@@ -69,35 +78,35 @@ func TestExplain(t *testing.T) {
"3.6.2",
}
samples := map[string]string{
"aggregate": "Cannot explain cmd: aggregate",
"count": "<nil>",
"count_with_query": "<nil>",
"delete": "<nil>",
"delete_all": "<nil>",
"distinct": "<nil>",
"find_empty": "<nil>",
"find": "<nil>",
"find_with_sort": "<nil>",
"find_andrii": "<nil>",
"findandmodify": "<nil>",
"geonear": "Cannot explain cmd: geoNear",
"getmore": "<nil>",
"group": "<nil>",
"insert": "Cannot explain cmd: insert",
"mapreduce": "Cannot explain cmd: mapReduce",
"update": "<nil>",
"explain": "Cannot explain cmd: explain",
"eval": "Cannot explain cmd: eval",
samples := map[string]bool{
"aggregate": false,
"count": false,
"count_with_query": false,
"delete": false,
"delete_all": false,
"distinct": false,
"find_empty": true,
"find": false,
"find_with_sort": false,
"find_andrii": false,
"findandmodify": false,
"geonear": true,
"getmore": false,
"group": false,
"insert": true,
"mapreduce": true,
"update": false,
"explain": true,
"eval": true,
}
expectError := map[string]string{}
expectError := map[string]bool{}
// For versions < 3.0 explain is not supported
if ok, _ := Constraint("< 3.0", bi.Version); ok {
for _, v := range versions {
for sample := range samples {
expectError[sample+"_"+v] = "no such cmd: explain"
expectError[sample+"_"+v] = true
}
}
} else {
@@ -110,27 +119,37 @@ func TestExplain(t *testing.T) {
for _, v := range versions {
// For versions < 3.4 parsing "getmore" is not supported and returns error
if ok, _ := Constraint("< 3.4", v); ok {
expectError["getmore_"+v] = "Explain failed due to unknown command: getmore"
expectError["getmore_"+v] = true
}
}
for _, v := range versions {
// For versions < 3.4 parsing "getmore" is not supported and returns error
if ok, _ := Constraint(">= 2.4, <= 2.6", v); ok {
expectError["find_empty_"+v] = false
}
if ok, _ := Constraint(">= 3.2", v); ok {
expectError["find_empty_"+v] = false
}
}
// For versions >= 3.0, < 3.4 trying to explain "insert" returns different error
if ok, _ := Constraint(">= 3.0, < 3.4", bi.Version); ok {
for _, v := range versions {
expectError["insert_"+v] = "Only update and delete write ops can be explained"
expectError["insert_"+v] = true
}
}
// Explaining `distinct` and `findAndModify` was introduced in MongoDB 3.2
if ok, _ := Constraint(">= 3.0, < 3.2", bi.Version); ok {
for _, v := range versions {
expectError["distinct_"+v] = "Cannot explain cmd: distinct"
expectError["findandmodify_"+v] = "Cannot explain cmd: findAndModify"
expectError["distinct_"+v] = true
expectError["findandmodify_"+v] = true
}
}
}
ex := New(session)
ex := New(ctx, client)
for _, file := range files {
t.Run(file.Name(), func(t *testing.T) {
eq := proto.ExampleQuery{}
@@ -144,9 +163,8 @@ func TestExplain(t *testing.T) {
}
got, err := ex.Explain("", query)
expectErrMsg := expectError[file.Name()]
gotErrMsg := fmt.Sprintf("%v", err)
if gotErrMsg != expectErrMsg {
t.Fatalf("explain error should be '%s' but was '%s'", expectErrMsg, gotErrMsg)
if (err != nil) != expectErrMsg {
t.Fatalf("explain error for %q \n %s\nshould be '%v' but was '%v'", string(query), file.Name(), expectErrMsg, err)
}
if err == nil {

View File

@@ -2,9 +2,9 @@
export DEBUG=1
export GOLANG_DOCKERHUB_TAG=1.10-stretch
export AWS_REGION=us-west-2
#export TEST_MONGODB_FLAVOR=percona/percona-server-mongodb
export TEST_MONGODB_FLAVOR=mongo
export TEST_PSMDB_VERSION=4.0
export TEST_MONGODB_FLAVOR=percona/percona-server-mongodb
#export TEST_MONGODB_FLAVOR=mongo
export TEST_PSMDB_VERSION=3.6
export TEST_MONGODB_ADMIN_USERNAME=admin
export TEST_MONGODB_ADMIN_PASSWORD=admin123456
export TEST_MONGODB_USERNAME=test