Updated deps

Gopkg.lock

@@ -9,6 +9,14 @@
   revision = "c7af12943936e8c39859482e61f0574c2fd7fc75"
   version = "v1.4.2"
 
+[[projects]]
+  digest = "1:c39fbf3b3e138accc03357c72417c0153c54cc1ae8c9f40e8f120a550d876a76"
+  name = "github.com/Percona-Lab/pt-pg-summary"
+  packages = ["models"]
+  pruneopts = ""
+  revision = "f06beea959eb00acfe44ce39342c27582ad84caa"
+  version = "v0.1.9"
+
 [[projects]]
   digest = "1:f82b8ac36058904227087141017bb82f4b0fc58272990a4cdae3e2d6d222644e"
   name = "github.com/StackExchange/wmi"
@@ -81,11 +89,11 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:6a6322a15aa8e99bd156fbba0aae4e5d67b4bb05251d860b348a45dfdcba9cce"
+  digest = "1:c823f556d11763bb805a60a77226ae321843c9bd1454ee5b808ca169ac515762"
   name = "github.com/golang/snappy"
   packages = ["."]
   pruneopts = ""
-  revision = "2a8bb927dd31d8daada140a5d09578521ce5c36a"
+  revision = "ff6b7dc882cf4cfba7ee0b9f7dcc1ac096c554aa"
 
 [[projects]]
   branch = "master"
@@ -165,11 +173,11 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:1adb91baf59317a1614c3b465b2734066c256d1dca99d5526baa43274542a737"
+  digest = "1:457024f04029bb321d759cc3b2c46b7e0e43572e3a663ce7d006aeb41efa2b17"
   name = "github.com/percona/go-mysql"
   packages = ["query"]
   pruneopts = ""
-  revision = "c5d0b4a3add9c9bf5bb26b2ab823289e395a3f98"
+  revision = "197f4ad8db8d1b04ff408042119176907c971f0a"
 
 [[projects]]
   digest = "1:1d7e1867c49a6dd9856598ef7c3123604ea3daabf5b83f303ff457bcbc410b1d"
@@ -188,7 +196,7 @@
   version = "v1.2.0"
 
 [[projects]]
-  digest = "1:2226ffdae873216a5bc8a0bab7a51ac670b27a4aed852007d77600f809aa04e3"
+  digest = "1:55dcddb2ba6ab25098ee6b96f176f39305f1fde7ea3d138e7e10bb64a5bf45be"
   name = "github.com/shirou/gopsutil"
   packages = [
     "cpu",
@@ -198,8 +206,8 @@
     "process",
   ]
   pruneopts = ""
-  revision = "d80c43f9c984a48783daf22f4bd9278006ae483a"
+  revision = "e4ec7b275ada47ca32799106c2dba142d96aaf93"
-  version = "v2.19.7"
+  version = "v2.19.8"
 
 [[projects]]
   branch = "master"
@@ -274,14 +282,14 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:a530f8e0c0ee8a3b440f9f0b0e9f4e5d5e47cfe3a581086ce32cd8ba114ddf4f"
+  digest = "1:2f8d339c3b89d5abf9a78aafe1e9fbe548f3b1fb9be5c3117036940904d39527"
   name = "golang.org/x/crypto"
   packages = [
     "pbkdf2",
     "ssh/terminal",
   ]
   pruneopts = ""
-  revision = "9756ffdc24725223350eb3266ffb92590d28f278"
+  revision = "71b5226ff73902d121cd9dbbdfdb67045a805845"
 
 [[projects]]
   branch = "master"
@@ -293,14 +301,14 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:d9222a165eab05f8b8f085c68fdee5ecc670f4f834e3ecbac6069dd3b768a6b3"
+  digest = "1:ffaa20332022643a821848aa2322787bbfbf06bceb4b4e84cde3b05d07fa51ac"
   name = "golang.org/x/sys"
   packages = [
     "unix",
     "windows",
   ]
   pruneopts = ""
-  revision = "c7b8b68b14567162c6602a7c5659ee0f26417c18"
+  revision = "749cb33beabd9aa6d3178e3de05bcc914f70b2bf"
 
 [[projects]]
   digest = "1:740b51a55815493a8d0f2b1e0d0ae48fe48953bf7eaf3fcc4198823bf67768c0"
@@ -322,6 +330,7 @@
   analyzer-version = 1
   input-imports = [
     "github.com/Masterminds/semver",
+    "github.com/Percona-Lab/pt-pg-summary/models",
     "github.com/alecthomas/kingpin",
     "github.com/go-ini/ini",
     "github.com/golang/mock/gomock",

Gopkg.toml

@@ -44,10 +44,6 @@
   branch = "master"
   name = "github.com/pborman/getopt"
 
-[[constraint]]
-  name = "github.com/percona/pmgo"
-  version = "0.5.1"
-
 [[constraint]]
   name = "github.com/pkg/errors"
   version = "0.8.0"
@@ -64,10 +60,6 @@
   name = "github.com/sirupsen/logrus"
   version = "1.0.4"
 
-[[constraint]]
-  branch = "v2"
-  name = "gopkg.in/mgo.v2"
-
 [[constraint]]
   name = "go.mongodb.org/mongo-driver"
   version = "~1.0.0"

src/go/lib/pginfo/pginfo.go

@@ -0,0 +1,324 @@
+package pginfo
+
+import (
+	"fmt"
+	"regexp"
+	"time"
+
+	"github.com/Percona-Lab/pt-pg-summary/models"
+	"github.com/hashicorp/go-version"
+	"github.com/pkg/errors"
+	"github.com/shirou/gopsutil/process"
+	"github.com/sirupsen/logrus"
+)
+
+// Process contains PostgreSQL process information
+type Process struct {
+	PID     int32
+	CmdLine string
+}
+
+// PGInfo has exported fields containing the data collected.
+// Fields are exported to be able to use them when printing the templates
+type PGInfo struct {
+	ClusterInfo        []*models.ClusterInfo
+	ConnectedClients   []*models.ConnectedClients
+	DatabaseWaitEvents []*models.DatabaseWaitEvents
+	AllDatabases       []*models.Databases
+	GlobalWaitEvents   []*models.GlobalWaitEvents
+	PortAndDatadir     *models.PortAndDatadir
+	SlaveHosts96       []*models.SlaveHosts96
+	SlaveHosts10       []*models.SlaveHosts10
+	Tablespaces        []*models.Tablespaces
+	Settings           []*models.Setting
+	Counters           map[models.Name][]*models.Counters    // Counters per database
+	IndexCacheHitRatio map[string]*models.IndexCacheHitRatio // Indexes cache hit ratio per database
+	TableCacheHitRatio map[string]*models.TableCacheHitRatio // Tables cache hit ratio per database
+	TableAccess        map[string][]*models.TableAccess      // Table access per database
+	ServerVersion      *version.Version
+	Sleep              int
+	Processes          []Process
+
+	// This is the list of databases from where we should get Table Cache Hit, Index Cache Hits, etc.
+	// This field is being populated on the newData function depending on the cli parameters.
+	// If --databases was not specified, this array will have the list of ALL databases from the GetDatabases
+	// method in the models pkg
+	databases []string
+	logger    *logrus.Logger
+}
+
+// New returns a new PGInfo instance with a local logger instance
+func New(db models.XODB, databases []string, sleep int) (*PGInfo, error) {
+	return new(db, databases, sleep, logrus.New())
+}
+
+// NewWithLogger returns a new PGInfo instance with an external logger instance
+func NewWithLogger(db models.XODB, databases []string, sleep int, l *logrus.Logger) (*PGInfo, error) {
+	return new(db, databases, sleep, l)
+}
+
+func new(db models.XODB, databases []string, sleep int, logger *logrus.Logger) (*PGInfo, error) {
+	var err error
+	info := &PGInfo{
+		databases:          databases,
+		Counters:           make(map[models.Name][]*models.Counters),
+		TableAccess:        make(map[string][]*models.TableAccess),
+		TableCacheHitRatio: make(map[string]*models.TableCacheHitRatio),
+		IndexCacheHitRatio: make(map[string]*models.IndexCacheHitRatio),
+		Sleep:              sleep,
+		logger:             logger,
+	}
+
+	if info.AllDatabases, err = models.GetDatabases(db); err != nil {
+		return nil, errors.Wrap(err, "Cannot get databases list")
+	}
+	info.logger.Debug("All databases list")
+	for i, db := range info.AllDatabases {
+		logger.Debugf("% 5d: %s", i, db.Datname)
+	}
+
+	if len(databases) < 1 {
+		info.databases = make([]string, 0, len(info.AllDatabases))
+		allDatabases, err := models.GetAllDatabases(db)
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot get the list of all databases")
+		}
+		for _, database := range allDatabases {
+			info.databases = append(info.databases, string(database.Datname))
+		}
+	} else {
+		info.databases = make([]string, len(databases))
+		copy(info.databases, databases)
+	}
+	info.logger.Debugf("Will collect info for these databases: %v", info.databases)
+
+	serverVersion, err := models.GetServerVersion(db)
+	if err != nil {
+		return nil, errors.Wrap(err, "Cannot get the connected clients list")
+	}
+
+	if info.ServerVersion, err = parseServerVersion(serverVersion.Version); err != nil {
+		return nil, fmt.Errorf("cannot get server version: %s", err.Error())
+	}
+	info.logger.Infof("Detected PostgreSQL version: %v", info.ServerVersion)
+
+	return info, nil
+}
+
+// DatabaseNames returns the list of the database names for which information will be collected
+func (i *PGInfo) DatabaseNames() []string {
+	return i.databases
+}
+
+// CollectPerDatabaseInfo collects information for a specific database
+func (i *PGInfo) CollectPerDatabaseInfo(db models.XODB, dbName string) (err error) {
+	i.logger.Info("Collecting Table Access information")
+	if i.TableAccess[dbName], err = models.GetTableAccesses(db); err != nil {
+		return errors.Wrapf(err, "cannot get Table Accesses for the %s ibase", dbName)
+	}
+
+	i.logger.Info("Collecting Table Cache Hit Ratio information")
+	if i.TableCacheHitRatio[dbName], err = models.GetTableCacheHitRatio(db); err != nil {
+		return errors.Wrapf(err, "cannot get Table Cache Hit Ratios for the %s ibase", dbName)
+	}
+
+	i.logger.Info("Collecting Index Cache Hit Ratio information")
+	if i.IndexCacheHitRatio[dbName], err = models.GetIndexCacheHitRatio(db); err != nil {
+		return errors.Wrapf(err, "cannot get Index Cache Hit Ratio for the %s ibase", dbName)
+	}
+
+	return nil
+}
+
+// CollectGlobalInfo collects global information
+func (i *PGInfo) CollectGlobalInfo(db models.XODB) []error {
+	errs := make([]error, 0)
+	var err error
+
+	version10, _ := version.NewVersion("10.0.0")
+
+	ch := make(chan interface{}, 2)
+	i.logger.Info("Collecting global counters (1st pass)")
+	getCounters(db, ch)
+	c1, err := waitForCounters(ch)
+	if err != nil {
+		errs = append(errs, errors.Wrap(err, "Cannot get counters (1st run)"))
+	} else {
+		for _, counters := range c1 {
+			i.Counters[counters.Datname] = append(i.Counters[counters.Datname], counters)
+		}
+	}
+
+	go func() {
+		i.logger.Infof("Waiting %d seconds to read  counters", i.Sleep)
+		time.Sleep(time.Duration(i.Sleep) * time.Second)
+		i.logger.Info("Collecting global counters (2nd pass)")
+		getCounters(db, ch)
+	}()
+
+	i.logger.Info("Collecting Cluster information")
+	if i.ClusterInfo, err = models.GetClusterInfos(db); err != nil {
+		errs = append(errs, errors.Wrap(err, "Cannot get cluster info"))
+	}
+
+	i.logger.Info("Collecting Connected Clients information")
+	if i.ConnectedClients, err = models.GetConnectedClients(db); err != nil {
+		errs = append(errs, errors.Wrap(err, "Cannot get the connected clients list"))
+	}
+
+	i.logger.Info("Collecting Database Wait Events information")
+	if i.DatabaseWaitEvents, err = models.GetDatabaseWaitEvents(db); err != nil {
+		errs = append(errs, errors.Wrap(err, "Cannot get databases wait events"))
+	}
+
+	i.logger.Info("Collecting Global Wait Events information")
+	if i.GlobalWaitEvents, err = models.GetGlobalWaitEvents(db); err != nil {
+		errs = append(errs, errors.Wrap(err, "Cannot get Global Wait Events"))
+	}
+
+	i.logger.Info("Collecting Port and Data Dir information")
+	if i.PortAndDatadir, err = models.GetPortAndDatadir(db); err != nil {
+		errs = append(errs, errors.Wrap(err, "Cannot get Port and Dir"))
+	}
+
+	i.logger.Info("Collecting Tablespaces information")
+	if i.Tablespaces, err = models.GetTablespaces(db); err != nil {
+		errs = append(errs, errors.Wrap(err, "Cannot get Tablespaces"))
+	}
+
+	i.logger.Info("Collecting Instance Settings information")
+	if i.Settings, err = models.GetSettings(db); err != nil {
+		errs = append(errs, errors.Wrap(err, "Cannot get instance settings"))
+	}
+
+	if i.ServerVersion.LessThan(version10) {
+		i.logger.Info("Collecting Slave Hosts (PostgreSQL < 10)")
+		if i.SlaveHosts96, err = models.GetSlaveHosts96s(db); err != nil {
+			errs = append(errs, errors.Wrap(err, "Cannot get slave hosts on Postgre < 10"))
+		}
+	}
+
+	if !i.ServerVersion.LessThan(version10) {
+		i.logger.Info("Collecting Slave Hosts (PostgreSQL 10+)")
+		if i.SlaveHosts10, err = models.GetSlaveHosts10s(db); err != nil {
+			errs = append(errs, errors.Wrap(err, "Cannot get slave hosts in Postgre 10+"))
+		}
+	}
+
+	i.logger.Info("Waiting for counters information")
+	c2, err := waitForCounters(ch)
+	if err != nil {
+		errs = append(errs, errors.Wrap(err, "Cannot read counters (2nd run)"))
+	} else {
+		for _, counters := range c2 {
+			i.Counters[counters.Datname] = append(i.Counters[counters.Datname], counters)
+		}
+		i.calcCountersDiff(i.Counters)
+	}
+
+	i.logger.Info("Collecting processes command line information")
+	if err := i.collectProcesses(); err != nil {
+		errs = append(errs, errors.Wrap(err, "Cannot collect processes information"))
+	}
+
+	i.logger.Info("Finished collecting global information")
+	return errs
+}
+
+// SetLogger sets an external logger instance
+func (i *PGInfo) SetLogger(l *logrus.Logger) {
+	i.logger = l
+}
+
+// SetLogLevel changes the current log level
+func (i *PGInfo) SetLogLevel(level logrus.Level) {
+	i.logger.SetLevel(level)
+}
+
+func getCounters(db models.XODB, ch chan interface{}) {
+	counters, err := models.GetCounters(db)
+	if err != nil {
+		ch <- err
+	} else {
+		ch <- counters
+	}
+}
+
+func waitForCounters(ch chan interface{}) ([]*models.Counters, error) {
+	resp := <-ch
+	if err, ok := resp.(error); ok {
+		return nil, err
+	}
+
+	return resp.([]*models.Counters), nil
+}
+
+func parseServerVersion(v string) (*version.Version, error) {
+	re := regexp.MustCompile(`(\d?\d)(\d\d)(\d\d)`)
+	m := re.FindStringSubmatch(v)
+	if len(m) != 4 {
+		return nil, fmt.Errorf("cannot parse version %s", v)
+	}
+	return version.NewVersion(fmt.Sprintf("%s.%s.%s", m[1], m[2], m[3]))
+}
+
+func (i *PGInfo) calcCountersDiff(counters map[models.Name][]*models.Counters) {
+	for dbName, c := range counters {
+		i.logger.Debugf("Calculating counters diff for %s database", dbName)
+		diff := &models.Counters{
+			Datname:      dbName,
+			Numbackends:  c[1].Numbackends - c[0].Numbackends,
+			XactCommit:   c[1].XactCommit - c[0].XactCommit,
+			XactRollback: c[1].XactRollback - c[0].XactRollback,
+			BlksRead:     c[1].BlksRead - c[0].BlksRead,
+			BlksHit:      c[1].BlksHit - c[0].BlksHit,
+			TupReturned:  c[1].TupReturned - c[0].TupReturned,
+			TupFetched:   c[1].TupFetched - c[0].TupFetched,
+			TupInserted:  c[1].TupInserted - c[0].TupInserted,
+			TupUpdated:   c[1].TupUpdated - c[0].TupUpdated,
+			TupDeleted:   c[1].TupDeleted - c[0].TupDeleted,
+			Conflicts:    c[1].Conflicts - c[0].Conflicts,
+			TempFiles:    c[1].TempFiles - c[0].TempFiles,
+			TempBytes:    c[1].TempBytes - c[0].TempBytes,
+			Deadlocks:    c[1].Deadlocks - c[0].Deadlocks,
+		}
+		counters[dbName] = append(counters[dbName], diff)
+		i.logger.Debugf("Numbackends : %v - %v", c[1].Numbackends, c[0].Numbackends)
+		i.logger.Debugf("XactCommit  : %v - %v", c[1].XactCommit, c[0].XactCommit)
+		i.logger.Debugf("XactRollback: %v - %v", c[1].XactRollback, c[0].XactRollback)
+		i.logger.Debugf("BlksRead    : %v - %v", c[1].BlksRead, c[0].BlksRead)
+		i.logger.Debugf("BlksHit     : %v - %v", c[1].BlksHit, c[0].BlksHit)
+		i.logger.Debugf("TupReturned : %v - %v", c[1].TupReturned, c[0].TupReturned)
+		i.logger.Debugf("TupFetched  : %v - %v", c[1].TupFetched, c[0].TupFetched)
+		i.logger.Debugf("TupInserted : %v - %v", c[1].TupInserted, c[0].TupInserted)
+		i.logger.Debugf("TupUpdated  : %v - %v", c[1].TupUpdated, c[0].TupUpdated)
+		i.logger.Debugf("TupDeleted  : %v - %v", c[1].TupDeleted, c[0].TupDeleted)
+		i.logger.Debugf("Conflicts   : %v - %v", c[1].Conflicts, c[0].Conflicts)
+		i.logger.Debugf("TempFiles   : %v - %v", c[1].TempFiles, c[0].TempFiles)
+		i.logger.Debugf("TempBytes   : %v - %v", c[1].TempBytes, c[0].TempBytes)
+		i.logger.Debugf("Deadlocks   : %v - %v", c[1].Deadlocks, c[0].Deadlocks)
+		i.logger.Debugf("---")
+	}
+}
+
+func (i *PGInfo) collectProcesses() error {
+	procs, err := process.Processes()
+	if err != nil {
+		return err
+	}
+
+	i.Processes = make([]Process, 0)
+
+	for _, proc := range procs {
+		cmdLine, err := proc.Cmdline()
+		if err != nil {
+			continue
+		}
+		match, _ := regexp.MatchString("^.*?/postgres\\s.*$", cmdLine)
+		if match {
+			i.Processes = append(i.Processes, Process{PID: proc.Pid, CmdLine: cmdLine})
+		}
+	}
+
+	return nil
+}