Merged pt-pg-summary

.gitignore

@@ -10,6 +10,8 @@ snapshot
 build
 Makefile.old
 bin/pt-mongodb-*
+bin/pt-secure*
+bin/pt-pg-summary
 !src/go/pt-mongodb-query-digest/vendor/vendor.json
 !src/go/pt-mongodb-summary/vendor/vendor.json
 src/go/pt-mongodb-query-digest/vendor/

Gopkg.lock

@@ -9,6 +9,14 @@
   revision = "c7af12943936e8c39859482e61f0574c2fd7fc75"
   version = "v1.4.2"
 
+[[projects]]
+  digest = "1:c39fbf3b3e138accc03357c72417c0153c54cc1ae8c9f40e8f120a550d876a76"
+  name = "github.com/Percona-Lab/pt-pg-summary"
+  packages = ["models"]
+  pruneopts = ""
+  revision = "f06beea959eb00acfe44ce39342c27582ad84caa"
+  version = "v0.1.9"
+
 [[projects]]
   digest = "1:f82b8ac36058904227087141017bb82f4b0fc58272990a4cdae3e2d6d222644e"
   name = "github.com/StackExchange/wmi"
@@ -128,12 +136,24 @@
   version = "v0.1.0"
 
 [[projects]]
-  digest = "1:0093a7c66d5b9e0cdaf4be5c20e0a9b889d1d839148eeed1d587e99b4cfd90ff"
+  digest = "1:f4216047c24ab66fb757045febd7dac4edc6f4ad9f6c0063d0755d654d04f25e"
+  name = "github.com/lib/pq"
+  packages = [
+    ".",
+    "oid",
+    "scram",
+  ]
+  pruneopts = ""
+  revision = "3427c32cb71afc948325f299f040e53c1dd78979"
+  version = "v1.2.0"
+
+[[projects]]
+  digest = "1:d5d962b7a8d95b9e6226f6b831a7e50237acf1730dcace6e8cb87c6dd628ef54"
   name = "github.com/mattn/go-shellwords"
   packages = ["."]
   pruneopts = ""
-  revision = "a72fbe27a1b0ed0df2f02754945044ce1456608b"
-  version = "v1.0.5"
+  revision = "36a9b3c57cb5caa559ff63fb7e9b585f1c00df75"
+  version = "v1.0.6"
 
 [[projects]]
   digest = "1:a067513044dc491395a58f56f39cedddb5ad35789b832b570c283a64d712f81b"
@@ -159,14 +179,6 @@
   pruneopts = ""
   revision = "c5d0b4a3add9c9bf5bb26b2ab823289e395a3f98"
 
-[[projects]]
-  digest = "1:16b4510ba61ab0bb7a4e694ea6396a7b2879f5fabb21e93066e182691f790173"
-  name = "github.com/percona/pmgo"
-  packages = ["."]
-  pruneopts = ""
-  revision = "497d06e28f910fbe26d5d60f59d36284a6901c6f"
-  version = "0.5.2"
-
 [[projects]]
   digest = "1:1d7e1867c49a6dd9856598ef7c3123604ea3daabf5b83f303ff457bcbc410b1d"
   name = "github.com/pkg/errors"
@@ -270,22 +282,14 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:086760278d762dbb0e9a26e09b57f04c89178c86467d8d94fae47d64c222f328"
+  digest = "1:00cb2d43e2f3a7b10653bbe965ca5b6ef074556adaea71a68b014a8f8cf566f7"
   name = "golang.org/x/crypto"
   packages = [
     "pbkdf2",
     "ssh/terminal",
   ]
   pruneopts = ""
-  revision = "4def268fd1a49955bfb3dda92fe3db4f924f2285"
-
-[[projects]]
-  branch = "master"
-  digest = "1:955694a7c42527d7fb188505a22f10b3e158c6c2cf31fe64b1e62c9ab7b18401"
-  name = "golang.org/x/net"
-  packages = ["context"]
-  pruneopts = ""
-  revision = "ca1201d0de80cfde86cb01aea620983605dfe99b"
+  revision = "60c769a6c58655dab1b9adac0d58967dd517cfba"
 
 [[projects]]
   branch = "master"
@@ -297,14 +301,14 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:0b5c2207c72f2d13995040f176feb6e3f453d6b01af2b9d57df76b05ded2e926"
+  digest = "1:6a8bd8cb9a92bd007831755e08d5f62810ecf6718e70cc38f0a85a91819a9497"
   name = "golang.org/x/sys"
   packages = [
     "unix",
     "windows",
   ]
   pruneopts = ""
-  revision = "51ab0e2deafac1f46c46ad59cf0921be2f180c3d"
+  revision = "fde4db37ae7ad8191b03d30d27f258b5291ae4e3"
 
 [[projects]]
   digest = "1:740b51a55815493a8d0f2b1e0d0ae48fe48953bf7eaf3fcc4198823bf67768c0"
@@ -321,45 +325,23 @@
   revision = "342b2e1fbaa52c93f31447ad2c6abc048c63e475"
   version = "v0.3.2"
 
-[[projects]]
-  branch = "v2"
-  digest = "1:f54ba71a035aac92ced3e902d2bff3734a15d1891daff73ec0f90ef236750139"
-  name = "gopkg.in/mgo.v2"
-  packages = [
-    ".",
-    "bson",
-    "dbtest",
-    "internal/json",
-    "internal/sasl",
-    "internal/scram",
-  ]
-  pruneopts = ""
-  revision = "9856a29383ce1c59f308dd1cf0363a79b5bef6b5"
-
-[[projects]]
-  branch = "v2"
-  digest = "1:61a650a53e5e865a91ae9581f02990a4b6e3afcb8d280f19b1e67a3c284944e6"
-  name = "gopkg.in/tomb.v2"
-  packages = ["."]
-  pruneopts = ""
-  revision = "d5d1b5820637886def9eef33e03a27a9f166942c"
-
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
   input-imports = [
     "github.com/Masterminds/semver",
+    "github.com/Percona-Lab/pt-pg-summary/models",
     "github.com/alecthomas/kingpin",
     "github.com/go-ini/ini",
     "github.com/golang/mock/gomock",
     "github.com/hashicorp/go-version",
     "github.com/howeyc/gopass",
     "github.com/kr/pretty",
+    "github.com/lib/pq",
     "github.com/mattn/go-shellwords",
     "github.com/montanaflynn/stats",
     "github.com/pborman/getopt",
     "github.com/percona/go-mysql/query",
-    "github.com/percona/pmgo",
     "github.com/pkg/errors",
     "github.com/satori/go.uuid",
     "github.com/shirou/gopsutil/process",

Gopkg.toml

@@ -44,10 +44,6 @@
   branch = "master"
   name = "github.com/pborman/getopt"
 
-[[constraint]]
-  name = "github.com/percona/pmgo"
-  version = "0.5.1"
-
 [[constraint]]
   name = "github.com/pkg/errors"
   version = "0.8.0"
@@ -64,10 +60,6 @@
   name = "github.com/sirupsen/logrus"
   version = "1.0.4"
 
-[[constraint]]
-  branch = "v2"
-  name = "gopkg.in/mgo.v2"
-
 [[constraint]]
   name = "go.mongodb.org/mongo-driver"
   version = "~1.0.0"

src/go/docker-compose.yml

@@ -143,6 +143,30 @@ services:
     - configsvr1
 #   - configsvr2
 #   - configsvr3
+  postgres9:
+    image: ${MYSQL_IMAGE:-postgres:9.6}
+    ports:
+      - ${POSTGRE_HOST:-127.0.0.1}:${POSTGRE_96_PORT:-6432}:5432
+    environment:
+      - POSTGRES_PASSWORD=root
+  postgres10:
+    image: ${POSTGRE_IMAGE:-postgres:10.7}
+    ports:
+      - ${POSTGRE_HOST:-127.0.0.1}:${POSTGRE_10_PORT:-6433}:5432
+    environment:
+      - POSTGRES_PASSWORD=root
+  postgres11:
+    image: ${POSTGRE_IMAGE:-postgres:11}
+    ports:
+      - ${POSTGRE_HOST:-127.0.0.1}:${POSTGRE_11_PORT:-6434}:5432
+    environment:
+      - POSTGRES_PASSWORD=root
+  postgres12:
+    image: ${POSTGRE_IMAGE:-postgres:12}
+    ports:
+      - ${POSTGRE_HOST:-127.0.0.1}:${POSTGRE_12_PORT:-6435}:5432
+    environment:
+      - POSTGRES_PASSWORD=root
   init:
     network_mode: host
     image: ${TEST_MONGODB_FLAVOR}:${TEST_PSMDB_VERSION}
@@ -166,6 +190,10 @@ services:
     - s3-mongo2
     - s3-mongo3
     - standalone
+    - postgres9
+    - postgres10
+    - postgres11
+    - postgres12
   test:
     build:
       dockerfile: docker/test/Dockerfile

src/go/lib/pginfo/pginfo.go

@@ -0,0 +1,324 @@
+package pginfo
+
+import (
+	"fmt"
+	"regexp"
+	"time"
+
+	"github.com/Percona-Lab/pt-pg-summary/models"
+	"github.com/hashicorp/go-version"
+	"github.com/pkg/errors"
+	"github.com/shirou/gopsutil/process"
+	"github.com/sirupsen/logrus"
+)
+
+// Process contains PostgreSQL process information
+type Process struct {
+	PID     int32
+	CmdLine string
+}
+
+// PGInfo has exported fields containing the data collected.
+// Fields are exported to be able to use them when printing the templates
+type PGInfo struct {
+	ClusterInfo        []*models.ClusterInfo
+	ConnectedClients   []*models.ConnectedClients
+	DatabaseWaitEvents []*models.DatabaseWaitEvents
+	AllDatabases       []*models.Databases
+	GlobalWaitEvents   []*models.GlobalWaitEvents
+	PortAndDatadir     *models.PortAndDatadir
+	SlaveHosts96       []*models.SlaveHosts96
+	SlaveHosts10       []*models.SlaveHosts10
+	Tablespaces        []*models.Tablespaces
+	Settings           []*models.Setting
+	Counters           map[models.Name][]*models.Counters    // Counters per database
+	IndexCacheHitRatio map[string]*models.IndexCacheHitRatio // Indexes cache hit ratio per database
+	TableCacheHitRatio map[string]*models.TableCacheHitRatio // Tables cache hit ratio per database
+	TableAccess        map[string][]*models.TableAccess      // Table access per database
+	ServerVersion      *version.Version
+	Sleep              int
+	Processes          []Process
+
+	// This is the list of databases from where we should get Table Cache Hit, Index Cache Hits, etc.
+	// This field is being populated on the newData function depending on the cli parameters.
+	// If --databases was not specified, this array will have the list of ALL databases from the GetDatabases
+	// method in the models pkg
+	databases []string
+	logger    *logrus.Logger
+}
+
+// New returns a new PGInfo instance with a local logger instance
+func New(db models.XODB, databases []string, sleep int) (*PGInfo, error) {
+	return new(db, databases, sleep, logrus.New())
+}
+
+// NewWithLogger returns a new PGInfo instance with an external logger instance
+func NewWithLogger(db models.XODB, databases []string, sleep int, l *logrus.Logger) (*PGInfo, error) {
+	return new(db, databases, sleep, l)
+}
+
+func new(db models.XODB, databases []string, sleep int, logger *logrus.Logger) (*PGInfo, error) {
+	var err error
+	info := &PGInfo{
+		databases:          databases,
+		Counters:           make(map[models.Name][]*models.Counters),
+		TableAccess:        make(map[string][]*models.TableAccess),
+		TableCacheHitRatio: make(map[string]*models.TableCacheHitRatio),
+		IndexCacheHitRatio: make(map[string]*models.IndexCacheHitRatio),
+		Sleep:              sleep,
+		logger:             logger,
+	}
+
+	if info.AllDatabases, err = models.GetDatabases(db); err != nil {
+		return nil, errors.Wrap(err, "Cannot get databases list")
+	}
+	info.logger.Debug("All databases list")
+	for i, db := range info.AllDatabases {
+		logger.Debugf("% 5d: %s", i, db.Datname)
+	}
+
+	if len(databases) < 1 {
+		info.databases = make([]string, 0, len(info.AllDatabases))
+		allDatabases, err := models.GetAllDatabases(db)
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot get the list of all databases")
+		}
+		for _, database := range allDatabases {
+			info.databases = append(info.databases, string(database.Datname))
+		}
+	} else {
+		info.databases = make([]string, len(databases))
+		copy(info.databases, databases)
+	}
+	info.logger.Debugf("Will collect info for these databases: %v", info.databases)
+
+	serverVersion, err := models.GetServerVersion(db)
+	if err != nil {
+		return nil, errors.Wrap(err, "Cannot get the connected clients list")
+	}
+
+	if info.ServerVersion, err = parseServerVersion(serverVersion.Version); err != nil {
+		return nil, fmt.Errorf("cannot get server version: %s", err.Error())
+	}
+	info.logger.Infof("Detected PostgreSQL version: %v", info.ServerVersion)
+
+	return info, nil
+}
+
+// DatabaseNames returns the list of the database names for which information will be collected
+func (i *PGInfo) DatabaseNames() []string {
+	return i.databases
+}
+
+// CollectPerDatabaseInfo collects information for a specific database
+func (i *PGInfo) CollectPerDatabaseInfo(db models.XODB, dbName string) (err error) {
+	i.logger.Info("Collecting Table Access information")
+	if i.TableAccess[dbName], err = models.GetTableAccesses(db); err != nil {
+		return errors.Wrapf(err, "cannot get Table Accesses for the %s ibase", dbName)
+	}
+
+	i.logger.Info("Collecting Table Cache Hit Ratio information")
+	if i.TableCacheHitRatio[dbName], err = models.GetTableCacheHitRatio(db); err != nil {
+		return errors.Wrapf(err, "cannot get Table Cache Hit Ratios for the %s ibase", dbName)
+	}
+
+	i.logger.Info("Collecting Index Cache Hit Ratio information")
+	if i.IndexCacheHitRatio[dbName], err = models.GetIndexCacheHitRatio(db); err != nil {
+		return errors.Wrapf(err, "cannot get Index Cache Hit Ratio for the %s ibase", dbName)
+	}
+
+	return nil
+}
+
+// CollectGlobalInfo collects global information
+func (i *PGInfo) CollectGlobalInfo(db models.XODB) []error {
+	errs := make([]error, 0)
+	var err error
+
+	version10, _ := version.NewVersion("10.0.0")
+
+	ch := make(chan interface{}, 2)
+	i.logger.Info("Collecting global counters (1st pass)")
+	getCounters(db, ch)
+	c1, err := waitForCounters(ch)
+	if err != nil {
+		errs = append(errs, errors.Wrap(err, "Cannot get counters (1st run)"))
+	} else {
+		for _, counters := range c1 {
+			i.Counters[counters.Datname] = append(i.Counters[counters.Datname], counters)
+		}
+	}
+
+	go func() {
+		i.logger.Infof("Waiting %d seconds to read  counters", i.Sleep)
+		time.Sleep(time.Duration(i.Sleep) * time.Second)
+		i.logger.Info("Collecting global counters (2nd pass)")
+		getCounters(db, ch)
+	}()
+
+	i.logger.Info("Collecting Cluster information")
+	if i.ClusterInfo, err = models.GetClusterInfos(db); err != nil {
+		errs = append(errs, errors.Wrap(err, "Cannot get cluster info"))
+	}
+
+	i.logger.Info("Collecting Connected Clients information")
+	if i.ConnectedClients, err = models.GetConnectedClients(db); err != nil {
+		errs = append(errs, errors.Wrap(err, "Cannot get the connected clients list"))
+	}
+
+	i.logger.Info("Collecting Database Wait Events information")
+	if i.DatabaseWaitEvents, err = models.GetDatabaseWaitEvents(db); err != nil {
+		errs = append(errs, errors.Wrap(err, "Cannot get databases wait events"))
+	}
+
+	i.logger.Info("Collecting Global Wait Events information")
+	if i.GlobalWaitEvents, err = models.GetGlobalWaitEvents(db); err != nil {
+		errs = append(errs, errors.Wrap(err, "Cannot get Global Wait Events"))
+	}
+
+	i.logger.Info("Collecting Port and Data Dir information")
+	if i.PortAndDatadir, err = models.GetPortAndDatadir(db); err != nil {
+		errs = append(errs, errors.Wrap(err, "Cannot get Port and Dir"))
+	}
+
+	i.logger.Info("Collecting Tablespaces information")
+	if i.Tablespaces, err = models.GetTablespaces(db); err != nil {
+		errs = append(errs, errors.Wrap(err, "Cannot get Tablespaces"))
+	}
+
+	i.logger.Info("Collecting Instance Settings information")
+	if i.Settings, err = models.GetSettings(db); err != nil {
+		errs = append(errs, errors.Wrap(err, "Cannot get instance settings"))
+	}
+
+	if i.ServerVersion.LessThan(version10) {
+		i.logger.Info("Collecting Slave Hosts (PostgreSQL < 10)")
+		if i.SlaveHosts96, err = models.GetSlaveHosts96s(db); err != nil {
+			errs = append(errs, errors.Wrap(err, "Cannot get slave hosts on Postgre < 10"))
+		}
+	}
+
+	if !i.ServerVersion.LessThan(version10) {
+		i.logger.Info("Collecting Slave Hosts (PostgreSQL 10+)")
+		if i.SlaveHosts10, err = models.GetSlaveHosts10s(db); err != nil {
+			errs = append(errs, errors.Wrap(err, "Cannot get slave hosts in Postgre 10+"))
+		}
+	}
+
+	i.logger.Info("Waiting for counters information")
+	c2, err := waitForCounters(ch)
+	if err != nil {
+		errs = append(errs, errors.Wrap(err, "Cannot read counters (2nd run)"))
+	} else {
+		for _, counters := range c2 {
+			i.Counters[counters.Datname] = append(i.Counters[counters.Datname], counters)
+		}
+		i.calcCountersDiff(i.Counters)
+	}
+
+	i.logger.Info("Collecting processes command line information")
+	if err := i.collectProcesses(); err != nil {
+		errs = append(errs, errors.Wrap(err, "Cannot collect processes information"))
+	}
+
+	i.logger.Info("Finished collecting global information")
+	return errs
+}
+
+// SetLogger sets an external logger instance
+func (i *PGInfo) SetLogger(l *logrus.Logger) {
+	i.logger = l
+}
+
+// SetLogLevel changes the current log level
+func (i *PGInfo) SetLogLevel(level logrus.Level) {
+	i.logger.SetLevel(level)
+}
+
+func getCounters(db models.XODB, ch chan interface{}) {
+	counters, err := models.GetCounters(db)
+	if err != nil {
+		ch <- err
+	} else {
+		ch <- counters
+	}
+}
+
+func waitForCounters(ch chan interface{}) ([]*models.Counters, error) {
+	resp := <-ch
+	if err, ok := resp.(error); ok {
+		return nil, err
+	}
+
+	return resp.([]*models.Counters), nil
+}
+
+func parseServerVersion(v string) (*version.Version, error) {
+	re := regexp.MustCompile(`(\d?\d)(\d\d)(\d\d)`)
+	m := re.FindStringSubmatch(v)
+	if len(m) != 4 {
+		return nil, fmt.Errorf("cannot parse version %s", v)
+	}
+	return version.NewVersion(fmt.Sprintf("%s.%s.%s", m[1], m[2], m[3]))
+}
+
+func (i *PGInfo) calcCountersDiff(counters map[models.Name][]*models.Counters) {
+	for dbName, c := range counters {
+		i.logger.Debugf("Calculating counters diff for %s database", dbName)
+		diff := &models.Counters{
+			Datname:      dbName,
+			Numbackends:  c[1].Numbackends - c[0].Numbackends,
+			XactCommit:   c[1].XactCommit - c[0].XactCommit,
+			XactRollback: c[1].XactRollback - c[0].XactRollback,
+			BlksRead:     c[1].BlksRead - c[0].BlksRead,
+			BlksHit:      c[1].BlksHit - c[0].BlksHit,
+			TupReturned:  c[1].TupReturned - c[0].TupReturned,
+			TupFetched:   c[1].TupFetched - c[0].TupFetched,
+			TupInserted:  c[1].TupInserted - c[0].TupInserted,
+			TupUpdated:   c[1].TupUpdated - c[0].TupUpdated,
+			TupDeleted:   c[1].TupDeleted - c[0].TupDeleted,
+			Conflicts:    c[1].Conflicts - c[0].Conflicts,
+			TempFiles:    c[1].TempFiles - c[0].TempFiles,
+			TempBytes:    c[1].TempBytes - c[0].TempBytes,
+			Deadlocks:    c[1].Deadlocks - c[0].Deadlocks,
+		}
+		counters[dbName] = append(counters[dbName], diff)
+		i.logger.Debugf("Numbackends : %v - %v", c[1].Numbackends, c[0].Numbackends)
+		i.logger.Debugf("XactCommit  : %v - %v", c[1].XactCommit, c[0].XactCommit)
+		i.logger.Debugf("XactRollback: %v - %v", c[1].XactRollback, c[0].XactRollback)
+		i.logger.Debugf("BlksRead    : %v - %v", c[1].BlksRead, c[0].BlksRead)
+		i.logger.Debugf("BlksHit     : %v - %v", c[1].BlksHit, c[0].BlksHit)
+		i.logger.Debugf("TupReturned : %v - %v", c[1].TupReturned, c[0].TupReturned)
+		i.logger.Debugf("TupFetched  : %v - %v", c[1].TupFetched, c[0].TupFetched)
+		i.logger.Debugf("TupInserted : %v - %v", c[1].TupInserted, c[0].TupInserted)
+		i.logger.Debugf("TupUpdated  : %v - %v", c[1].TupUpdated, c[0].TupUpdated)
+		i.logger.Debugf("TupDeleted  : %v - %v", c[1].TupDeleted, c[0].TupDeleted)
+		i.logger.Debugf("Conflicts   : %v - %v", c[1].Conflicts, c[0].Conflicts)
+		i.logger.Debugf("TempFiles   : %v - %v", c[1].TempFiles, c[0].TempFiles)
+		i.logger.Debugf("TempBytes   : %v - %v", c[1].TempBytes, c[0].TempBytes)
+		i.logger.Debugf("Deadlocks   : %v - %v", c[1].Deadlocks, c[0].Deadlocks)
+		i.logger.Debugf("---")
+	}
+}
+
+func (i *PGInfo) collectProcesses() error {
+	procs, err := process.Processes()
+	if err != nil {
+		return err
+	}
+
+	i.Processes = make([]Process, 0)
+
+	for _, proc := range procs {
+		cmdLine, err := proc.Cmdline()
+		if err != nil {
+			continue
+		}
+		match, _ := regexp.MatchString("^.*?/postgres\\s.*$", cmdLine)
+		if match {
+			i.Processes = append(i.Processes, Process{PID: proc.Pid, CmdLine: cmdLine})
+		}
+	}
+
+	return nil
+}