pt-galera-log-explainer: add whois command

src/go/pt-galera-log-explainer/main.go

@@ -39,8 +39,8 @@ var CLI struct {
 	MergeByDirectory      bool            `help:"Instead of relying on identification, merge contexts and columns by base directory. Very useful when dealing with many small logs organized per directories."`
 	SkipMerge             bool            `help:"Disable the ability to merge log files together. Can be used when every nodes have the same wsrep_node_name"`
 
-	List list `cmd:""`
+	List  list  `cmd:""`
-	//Whois whois `cmd:""`
+	Whois whois `cmd:""`
 	//	Sed       sed       `cmd:""`
 	Ctx       ctx       `cmd:""`
 	RegexList regexList `cmd:""`

src/go/pt-galera-log-explainer/regex/regex.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/types"
+	"github.com/rs/zerolog/log"
 )
 
 func internalRegexSubmatch(regex *regexp.Regexp, log string) ([]string, error) {
@@ -41,32 +42,56 @@ func AllRegexes() types.RegexMap {
 // general building block wsrep regexes
 // It's later used to identify subgroups easier
 var (
-	groupMethod        = "ssltcp"
+	groupMethod       = "ssltcp"
-	groupNodeIP        = "nodeip"
+	groupNodeIP       = "nodeip"
-	groupNodeHash      = "uuid"
+	groupNodeHash     = "uuid"
-	groupUUID          = "uuid" // same value as groupnodehash, because both are used in same context
+	groupUUID         = "uuid" // same value as groupnodehash, because both are used in same context
-	groupNodeName      = "nodename"
+	groupNodeName     = "nodename"
-	groupNodeName2     = "nodename2"
+	groupNodeName2    = "nodename2"
-	groupIdx           = "idx"
+	groupIdx          = "idx"
-	groupSeqno         = "seqno"
+	groupSeqno        = "seqno"
-	groupMembers       = "members"
+	groupMembers      = "members"
-	groupVersion       = "version"
+	groupVersion      = "version"
-	groupErrorMD5      = "errormd5"
+	groupErrorMD5     = "errormd5"
-	regexMembers       = "(?P<" + groupMembers + ">[0-9]{1,2})"
+	regexMembers      = "(?P<" + groupMembers + ">[0-9]{1,2})"
-	regexNodeHash      = "(?P<" + groupNodeHash + ">[a-zA-Z0-9-_]+)"
+	regexNodeHash     = "(?P<" + groupNodeHash + ">[a-zA-Z0-9-_]+)"
-	regexNodeName      = "(?P<" + groupNodeName + `>[a-zA-Z0-9-_\.]+)`
+	regexNodeName     = "(?P<" + groupNodeName + `>[a-zA-Z0-9-_\.]+)`
-	regexNodeName2     = strings.Replace(regexNodeName, groupNodeName, groupNodeName2, 1)
+	regexNodeName2    = strings.Replace(regexNodeName, groupNodeName, groupNodeName2, 1)
-	regexUUID          = "(?P<" + groupUUID + ">[a-z0-9]+-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]+)" // eg ed97c863-d5c9-11ec-8ab7-671bbd2d70ef
+	regexUUID         = "(?P<" + groupUUID + ">[a-z0-9]+-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]+)" // eg ed97c863-d5c9-11ec-8ab7-671bbd2d70ef
-	regexNodeHash1Dash = "(?P<" + groupNodeHash + ">[a-z0-9]+-[a-z0-9]{4})"                               // eg ed97c863-8ab7
+	regexShortUUID    = "(?P<" + groupUUID + ">[a-z0-9]+-[a-z0-9]{4})"                                   // eg ed97c863-8ab7
-	regexSeqno         = "(?P<" + groupSeqno + ">[0-9]+)"
+	regexSeqno        = "(?P<" + groupSeqno + ">[0-9]+)"
-	regexNodeIP        = "(?P<" + groupNodeIP + ">[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})"
+	regexNodeIP       = "(?P<" + groupNodeIP + ">[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})"
-	regexNodeIPMethod  = "(?P<" + groupMethod + ">.+)://" + regexNodeIP + ":[0-9]{1,6}"
+	regexNodeIPMethod = "(?P<" + groupMethod + ">.+)://" + regexNodeIP + ":[0-9]{1,6}"
-	regexIdx           = "(?P<" + groupIdx + ">-?[0-9]{1,2})(\\.-?[0-9])?"
+	regexIdx          = "(?P<" + groupIdx + ">-?[0-9]{1,2})(\\.-?[0-9])?"
-	regexVersion       = "(?P<" + groupVersion + ">(5|8|10|11)\\.[0-9]\\.[0-9]{1,2})"
+	regexVersion      = "(?P<" + groupVersion + ">(5|8|10|11)\\.[0-9]\\.[0-9]{1,2})"
-	regexErrorMD5      = "(?P<" + groupErrorMD5 + ">[a-z0-9]*)"
+	regexErrorMD5     = "(?P<" + groupErrorMD5 + ">[a-z0-9]*)"
 )
 
+// IsNodeUUID can only try to see if that's an UUID
+// functionally, it could also be a "regexNodeHash", but it's indistinguishable from wsrep_node_name
+// as it won't have any specific format
 func IsNodeUUID(s string) bool {
-	b, _ := regexp.MatchString(regexUUID, s)
+	b, err := regexp.MatchString(regexUUID, s)
+	if err != nil {
+		log.Warn().Err(err).Str("input", s).Msg("failed to check if it is an uuid")
+		return false
+	}
+	if b {
+		return true
+	}
+	b, err = regexp.MatchString(regexShortUUID, s)
+	if err != nil {
+		log.Warn().Err(err).Str("input", s).Msg("failed to check if it is a short uuid")
+		return false
+	}
+	return b
+}
+
+func IsNodeIP(s string) bool {
+	b, err := regexp.MatchString(regexNodeIP, s)
+	if err != nil {
+		log.Warn().Err(err).Str("input", s).Msg("failed to check if it is an ip")
+		return false
+	}
 	return b
 }

src/go/pt-galera-log-explainer/translate/translate.go

@@ -23,7 +23,7 @@ type translationsDB struct {
 	HashToNodeNames map[string][]translationUnit
 	IPToNodeNames   map[string][]translationUnit
 
-	// incase methods changed in the middle, tls=>ssl
+	// incase methods changed in the middle, tcp=>ssl
 	IPToMethods map[string][]translationUnit
 	rwlock      sync.RWMutex
 }
@@ -203,3 +203,37 @@ func SimplestInfoFromHash(hash string, date time.Time) string {
 	}
 	return hash
 }
+
+func IsNodeUUIDKnown(uuid string) bool {
+	db.rwlock.RLock()
+	defer db.rwlock.RUnlock()
+
+	_, ok := db.HashToIP[uuid]
+	if ok {
+		return true
+	}
+	_, ok = db.HashToNodeNames[uuid]
+	return ok
+}
+
+func IsNodeNameKnown(name string) bool {
+	db.rwlock.RLock()
+	defer db.rwlock.RUnlock()
+
+	for _, nodenames := range db.HashToNodeNames {
+		for _, nodename := range nodenames {
+			if name == nodename.Value {
+				return true
+			}
+		}
+	}
+	for _, nodenames := range db.IPToNodeNames {
+		for _, nodename := range nodenames {
+			if name == nodename.Value {
+				return true
+			}
+		}
+
+	}
+	return false
+}

src/go/pt-galera-log-explainer/translate/whois.go

@@ -0,0 +1,158 @@
+package translate
+
+import (
+	"encoding/json"
+	"time"
+)
+
+type WhoisNode struct {
+	parentNode *WhoisNode            `json:"-"`
+	rootNode   *WhoisNode            `json:"-"`
+	nodetype   string                `json:"-"`
+	Values     map[string]WhoisValue // the key here are the actual values stored for this node
+}
+
+type WhoisValue struct {
+	Timestamp *time.Time            `json:",omitempty"` // only the base one will be nil
+	SubNodes  map[string]*WhoisNode `json:",omitempty"` // associating the next node to a type of value (uuid, ip, node name)
+}
+
+type subNode map[string]*WhoisNode
+
+func Whois(search, searchtype string) *WhoisNode {
+	w := &WhoisNode{
+		nodetype: searchtype,
+		Values:   map[string]WhoisValue{},
+	}
+	w.rootNode = w
+	w.Values[search] = WhoisValue{SubNodes: map[string]*WhoisNode{}}
+	w.filter()
+	return w
+}
+
+func (v WhoisValue) AddChildKey(parentNode *WhoisNode, nodetype, value string, timestamp time.Time) {
+	child := v.SubNodes[nodetype]
+	nodeNew := false
+	if child == nil {
+		child = &WhoisNode{
+			nodetype:   nodetype,
+			rootNode:   parentNode.rootNode,
+			parentNode: parentNode,
+			Values:     map[string]WhoisValue{},
+		}
+		// delaying storage, we have to make sure
+		// not to store duplicate nodes first to avoid infinite recursion
+		nodeNew = true
+	}
+	ok := child.addKey(value, timestamp)
+	if nodeNew && ok {
+		v.SubNodes[nodetype] = child
+	}
+}
+
+func (n *WhoisNode) MarshalJSON() ([]byte, error) {
+	return json.Marshal(n.Values)
+}
+
+func (n *WhoisNode) addKey(value string, timestamp time.Time) bool {
+	storedValue := n.rootNode.GetValueData(value, n.nodetype)
+	if storedValue != nil {
+		if storedValue.Timestamp != nil && storedValue.Timestamp.Before(timestamp) {
+			storedValue.Timestamp = &timestamp
+		}
+		return false
+	}
+	n.Values[value] = WhoisValue{Timestamp: &timestamp, SubNodes: map[string]*WhoisNode{}}
+	return true
+}
+
+func (n *WhoisNode) GetValueData(search, searchType string) *WhoisValue {
+	for value, valueData := range n.Values {
+		if n.nodetype == searchType && search == value {
+			return &valueData
+		}
+		for _, nextNode := range valueData.SubNodes {
+			if nextNode != nil {
+				if valueData := nextNode.GetValueData(search, searchType); valueData != nil {
+					return valueData
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (n *WhoisNode) filter() {
+	switch n.nodetype {
+	case "ip":
+		n.filterDBUsingIP()
+	case "uuid":
+		n.FilterDBUsingUUID()
+	case "nodename":
+		n.FilterDBUsingNodeName()
+	}
+
+	for _, valueData := range n.Values {
+		for _, nextNode := range valueData.SubNodes {
+			if nextNode != nil {
+				nextNode.filter()
+			}
+		}
+	}
+}
+
+func (n *WhoisNode) filterDBUsingIP() {
+	for ip, valueData := range n.Values {
+		for hash, ip2 := range db.HashToIP {
+			if ip == ip2.Value {
+				valueData.AddChildKey(n, "uuid", hash, ip2.Timestamp)
+			}
+		}
+		nodenames, ok := db.IPToNodeNames[ip]
+		if ok {
+			for _, nodename := range nodenames {
+				valueData.AddChildKey(n, "nodename", nodename.Value, nodename.Timestamp)
+			}
+		}
+	}
+
+	return
+}
+
+func (n *WhoisNode) FilterDBUsingUUID() {
+	for uuid, valueData := range n.Values {
+		nodenames, ok := db.HashToNodeNames[uuid]
+		if ok {
+			for _, nodename := range nodenames {
+				valueData.AddChildKey(n, "nodename", nodename.Value, nodename.Timestamp)
+			}
+		}
+		ip, ok := db.HashToIP[uuid]
+		if ok {
+			valueData.AddChildKey(n, "ip", ip.Value, ip.Timestamp)
+		}
+	}
+
+	return
+}
+
+func (n *WhoisNode) FilterDBUsingNodeName() {
+	for nodename, valueData := range n.Values {
+		for uuid, nodenames2 := range db.HashToNodeNames {
+			for _, nodename2 := range nodenames2 {
+				if nodename == nodename2.Value {
+					valueData.AddChildKey(n, "uuid", uuid, nodename2.Timestamp)
+				}
+			}
+		}
+		for ip, nodenames2 := range db.IPToNodeNames {
+			for _, nodename2 := range nodenames2 {
+				if nodename == nodename2.Value {
+					valueData.AddChildKey(n, "ip", ip, nodename2.Timestamp)
+				}
+			}
+		}
+	}
+
+	return
+}

src/go/pt-galera-log-explainer/types/nodeinfo.go

@@ -1,12 +1,8 @@
 package types
 
-// NodeInfo is mainly used by "whois" subcommand
+type WhoisOutput struct {
-// This is to display its result
-// As it's the base work for "sed" subcommand, it's in types package
-type NodeInfo struct {
 	Input     string   `json:"input"`
 	IPs       []string `json:"IPs"`
 	NodeNames []string `json:"nodeNames"`
-	Hostname  string   `json:"hostname"`
 	NodeUUIDs []string `json:"nodeUUIDs:"`
 }

src/go/pt-galera-log-explainer/whois.go

@@ -1,94 +1,70 @@
 package main
 
-/*
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/regex"
+	"github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/translate"
+	"github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/utils"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+)
+
 type whois struct {
-	Search string   `arg:"" name:"search" help:"the identifier (node name, ip, uuid, hash) to search"`
+	Search     string   `arg:"" name:"search" help:"the identifier (node name, ip, uuid) to search"`
-	Paths  []string `arg:"" name:"paths" help:"paths of the log to use"`
+	SearchType string   `name:"type" help:"what kind of information is the input (node name, ip, uuid). Auto-detected when possible." enum:"nodename,ip,uuid,auto" default:"auto"`
+	Paths      []string `arg:"" name:"paths" help:"paths of the log to use"`
 }
 
 func (w *whois) Help() string {
 	return `Take any type of info pasted from error logs and find out about it.
-It will list known node name(s), IP(s), hostname(s), and other known node's UUIDs.
+It will list known node name(s), IP(s), and other known node's UUIDs.
+
+Regarding UUIDs (wsrep_gcomm_uuid), different format can be found in logs depending on versions : 
+- UUID, example: ac0f3910-9790-486c-afd4-845d0ae95692 
+- short UUID, with only 1st and 4st part: ac0f3910-afd4
+- shortest UUID, with only the 1st part: ac0f3910
 `
 }
 
 func (w *whois) Run() error {
 
-	toCheck := regex.AllRegexes()
+	if w.SearchType == "auto" {
-	timeline, err := timelineFromPaths(CLI.Whois.Paths, toCheck)
+		if regex.IsNodeUUID(w.Search) {
+			w.Search = utils.UUIDToShortUUID(w.Search)
+			w.SearchType = "uuid"
+		} else if regex.IsNodeIP(w.Search) {
+			w.SearchType = "ip"
+		} else if len(w.Search) != 8 { // at this point it's only a doubt between names and legacy node uuid, where only the first part of the uuid was shown in log
+			w.SearchType = "nodename"
+		} else {
+			log.Info().Msg("input information's type is ambiguous, scanning files to discover the type. You can also provide --type to avoid auto-detection")
+		}
+	}
+
+	_, err := timelineFromPaths(CLI.Whois.Paths, regex.AllRegexes())
 	if err != nil {
 		return errors.Wrap(err, "found nothing to translate")
 	}
-	ctxs := timeline.GetLatestContextsByNodes()
 
-	ni := whoIs(ctxs, CLI.Whois.Search)
+	if w.SearchType == "auto" {
+		if translate.IsNodeUUIDKnown(w.Search) {
+			w.SearchType = "uuid"
+		} else if translate.IsNodeNameKnown(w.Search) {
+			w.SearchType = "nodename"
+		} else {
+			return errors.New("could not detect the type of input. Try to provide --type. It may means the info is unknown")
+		}
+	}
 
-	json, err := json.MarshalIndent(ni, "", "\t")
+	log.Debug().Str("searchType", w.SearchType).Msg("whois searchType")
+	out := translate.Whois(w.Search, w.SearchType)
+
+	json, err := json.MarshalIndent(out, "", "\t")
 	if err != nil {
 		return err
 	}
 	fmt.Println(string(json))
 	return nil
 }
-
-func whoIs(ctxs map[string]types.LogCtx, search string) types.NodeInfo {
-		ni := types.NodeInfo{Input: search}
-		if regex.IsNodeUUID(search) {
-			search = utils.UUIDToShortUUID(search)
-		}
-		var (
-			ips       []string
-			hashes    []string
-			nodenames []string
-		)
-			for _, ctx := range ctxs {
-				if utils.SliceContains(ctx.OwnNames, search) || utils.SliceContains(ctx.OwnHashes, search) || utils.SliceContains(ctx.OwnIPs, search) {
-					ni.NodeNames = ctx.OwnNames
-					ni.NodeUUIDs = ctx.OwnHashes
-					ni.IPs = ctx.OwnIPs
-					ni.Hostname = ctx.OwnHostname()
-				}
-
-				if nodename, ok := ctx.HashToNodeName[search]; ok {
-					nodenames = utils.SliceMergeDeduplicate(nodenames, []string{nodename})
-					hashes = utils.SliceMergeDeduplicate(hashes, []string{search})
-				}
-
-				if ip, ok := ctx.HashToIP[search]; ok {
-					ips = utils.SliceMergeDeduplicate(ips, []string{ip})
-					hashes = utils.SliceMergeDeduplicate(hashes, []string{search})
-
-				} else if nodename, ok := ctx.IPToNodeName[search]; ok {
-					nodenames = utils.SliceMergeDeduplicate(nodenames, []string{nodename})
-					ips = utils.SliceMergeDeduplicate(ips, []string{search})
-
-				} else if utils.SliceContains(ctx.AllNodeNames(), search) {
-					nodenames = utils.SliceMergeDeduplicate(nodenames, []string{search})
-				}
-
-				for _, nodename := range nodenames {
-					hashes = utils.SliceMergeDeduplicate(hashes, ctx.HashesFromNodeName(nodename))
-					ips = utils.SliceMergeDeduplicate(ips, ctx.IPsFromNodeName(nodename))
-				}
-
-				for _, ip := range ips {
-					hashes = utils.SliceMergeDeduplicate(hashes, ctx.HashesFromIP(ip))
-					nodename, ok := ctx.IPToNodeName[ip]
-					if ok {
-						nodenames = utils.SliceMergeDeduplicate(nodenames, []string{nodename})
-					}
-				}
-				for _, hash := range hashes {
-					nodename, ok := ctx.HashToNodeName[hash]
-					if ok {
-						nodenames = utils.SliceMergeDeduplicate(nodenames, []string{nodename})
-					}
-				}
-			}
-			ni.NodeNames = nodenames
-			ni.NodeUUIDs = hashes
-			ni.IPs = ips
-			return ni
-	return types.NodeInfo{}
-}
-*/