diff --git a/src/go/pt-galera-log-explainer/main.go b/src/go/pt-galera-log-explainer/main.go index 7e78db1f..f872bb0c 100644 --- a/src/go/pt-galera-log-explainer/main.go +++ b/src/go/pt-galera-log-explainer/main.go @@ -39,8 +39,8 @@ var CLI struct { MergeByDirectory bool `help:"Instead of relying on identification, merge contexts and columns by base directory. Very useful when dealing with many small logs organized per directories."` SkipMerge bool `help:"Disable the ability to merge log files together. Can be used when every nodes have the same wsrep_node_name"` - List list `cmd:""` - //Whois whois `cmd:""` + List list `cmd:""` + Whois whois `cmd:""` // Sed sed `cmd:""` Ctx ctx `cmd:""` RegexList regexList `cmd:""` diff --git a/src/go/pt-galera-log-explainer/regex/regex.go b/src/go/pt-galera-log-explainer/regex/regex.go index 44f8210c..e21e37e9 100644 --- a/src/go/pt-galera-log-explainer/regex/regex.go +++ b/src/go/pt-galera-log-explainer/regex/regex.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/types" + "github.com/rs/zerolog/log" ) func internalRegexSubmatch(regex *regexp.Regexp, log string) ([]string, error) { @@ -41,32 +42,56 @@ func AllRegexes() types.RegexMap { // general building block wsrep regexes // It's later used to identify subgroups easier var ( - groupMethod = "ssltcp" - groupNodeIP = "nodeip" - groupNodeHash = "uuid" - groupUUID = "uuid" // same value as groupnodehash, because both are used in same context - groupNodeName = "nodename" - groupNodeName2 = "nodename2" - groupIdx = "idx" - groupSeqno = "seqno" - groupMembers = "members" - groupVersion = "version" - groupErrorMD5 = "errormd5" - regexMembers = "(?P<" + groupMembers + ">[0-9]{1,2})" - regexNodeHash = "(?P<" + groupNodeHash + ">[a-zA-Z0-9-_]+)" - regexNodeName = "(?P<" + groupNodeName + `>[a-zA-Z0-9-_\.]+)` - regexNodeName2 = strings.Replace(regexNodeName, groupNodeName, groupNodeName2, 1) - regexUUID = "(?P<" + groupUUID + ">[a-z0-9]+-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]+)" // eg ed97c863-d5c9-11ec-8ab7-671bbd2d70ef - regexNodeHash1Dash = "(?P<" + groupNodeHash + ">[a-z0-9]+-[a-z0-9]{4})" // eg ed97c863-8ab7 - regexSeqno = "(?P<" + groupSeqno + ">[0-9]+)" - regexNodeIP = "(?P<" + groupNodeIP + ">[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})" - regexNodeIPMethod = "(?P<" + groupMethod + ">.+)://" + regexNodeIP + ":[0-9]{1,6}" - regexIdx = "(?P<" + groupIdx + ">-?[0-9]{1,2})(\\.-?[0-9])?" - regexVersion = "(?P<" + groupVersion + ">(5|8|10|11)\\.[0-9]\\.[0-9]{1,2})" - regexErrorMD5 = "(?P<" + groupErrorMD5 + ">[a-z0-9]*)" + groupMethod = "ssltcp" + groupNodeIP = "nodeip" + groupNodeHash = "uuid" + groupUUID = "uuid" // same value as groupnodehash, because both are used in same context + groupNodeName = "nodename" + groupNodeName2 = "nodename2" + groupIdx = "idx" + groupSeqno = "seqno" + groupMembers = "members" + groupVersion = "version" + groupErrorMD5 = "errormd5" + regexMembers = "(?P<" + groupMembers + ">[0-9]{1,2})" + regexNodeHash = "(?P<" + groupNodeHash + ">[a-zA-Z0-9-_]+)" + regexNodeName = "(?P<" + groupNodeName + `>[a-zA-Z0-9-_\.]+)` + regexNodeName2 = strings.Replace(regexNodeName, groupNodeName, groupNodeName2, 1) + regexUUID = "(?P<" + groupUUID + ">[a-z0-9]+-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]+)" // eg ed97c863-d5c9-11ec-8ab7-671bbd2d70ef + regexShortUUID = "(?P<" + groupUUID + ">[a-z0-9]+-[a-z0-9]{4})" // eg ed97c863-8ab7 + regexSeqno = "(?P<" + groupSeqno + ">[0-9]+)" + regexNodeIP = "(?P<" + groupNodeIP + ">[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})" + regexNodeIPMethod = "(?P<" + groupMethod + ">.+)://" + regexNodeIP + ":[0-9]{1,6}" + regexIdx = "(?P<" + groupIdx + ">-?[0-9]{1,2})(\\.-?[0-9])?" + regexVersion = "(?P<" + groupVersion + ">(5|8|10|11)\\.[0-9]\\.[0-9]{1,2})" + regexErrorMD5 = "(?P<" + groupErrorMD5 + ">[a-z0-9]*)" ) +// IsNodeUUID can only try to see if that's an UUID +// functionally, it could also be a "regexNodeHash", but it's indistinguishable from wsrep_node_name +// as it won't have any specific format func IsNodeUUID(s string) bool { - b, _ := regexp.MatchString(regexUUID, s) + b, err := regexp.MatchString(regexUUID, s) + if err != nil { + log.Warn().Err(err).Str("input", s).Msg("failed to check if it is an uuid") + return false + } + if b { + return true + } + b, err = regexp.MatchString(regexShortUUID, s) + if err != nil { + log.Warn().Err(err).Str("input", s).Msg("failed to check if it is a short uuid") + return false + } + return b +} + +func IsNodeIP(s string) bool { + b, err := regexp.MatchString(regexNodeIP, s) + if err != nil { + log.Warn().Err(err).Str("input", s).Msg("failed to check if it is an ip") + return false + } return b } diff --git a/src/go/pt-galera-log-explainer/translate/translate.go b/src/go/pt-galera-log-explainer/translate/translate.go index c5858d95..295833dc 100644 --- a/src/go/pt-galera-log-explainer/translate/translate.go +++ b/src/go/pt-galera-log-explainer/translate/translate.go @@ -23,7 +23,7 @@ type translationsDB struct { HashToNodeNames map[string][]translationUnit IPToNodeNames map[string][]translationUnit - // incase methods changed in the middle, tls=>ssl + // incase methods changed in the middle, tcp=>ssl IPToMethods map[string][]translationUnit rwlock sync.RWMutex } @@ -203,3 +203,37 @@ func SimplestInfoFromHash(hash string, date time.Time) string { } return hash } + +func IsNodeUUIDKnown(uuid string) bool { + db.rwlock.RLock() + defer db.rwlock.RUnlock() + + _, ok := db.HashToIP[uuid] + if ok { + return true + } + _, ok = db.HashToNodeNames[uuid] + return ok +} + +func IsNodeNameKnown(name string) bool { + db.rwlock.RLock() + defer db.rwlock.RUnlock() + + for _, nodenames := range db.HashToNodeNames { + for _, nodename := range nodenames { + if name == nodename.Value { + return true + } + } + } + for _, nodenames := range db.IPToNodeNames { + for _, nodename := range nodenames { + if name == nodename.Value { + return true + } + } + + } + return false +} diff --git a/src/go/pt-galera-log-explainer/translate/whois.go b/src/go/pt-galera-log-explainer/translate/whois.go new file mode 100644 index 00000000..762d4fe1 --- /dev/null +++ b/src/go/pt-galera-log-explainer/translate/whois.go @@ -0,0 +1,158 @@ +package translate + +import ( + "encoding/json" + "time" +) + +type WhoisNode struct { + parentNode *WhoisNode `json:"-"` + rootNode *WhoisNode `json:"-"` + nodetype string `json:"-"` + Values map[string]WhoisValue // the key here are the actual values stored for this node +} + +type WhoisValue struct { + Timestamp *time.Time `json:",omitempty"` // only the base one will be nil + SubNodes map[string]*WhoisNode `json:",omitempty"` // associating the next node to a type of value (uuid, ip, node name) +} + +type subNode map[string]*WhoisNode + +func Whois(search, searchtype string) *WhoisNode { + w := &WhoisNode{ + nodetype: searchtype, + Values: map[string]WhoisValue{}, + } + w.rootNode = w + w.Values[search] = WhoisValue{SubNodes: map[string]*WhoisNode{}} + w.filter() + return w +} + +func (v WhoisValue) AddChildKey(parentNode *WhoisNode, nodetype, value string, timestamp time.Time) { + child := v.SubNodes[nodetype] + nodeNew := false + if child == nil { + child = &WhoisNode{ + nodetype: nodetype, + rootNode: parentNode.rootNode, + parentNode: parentNode, + Values: map[string]WhoisValue{}, + } + // delaying storage, we have to make sure + // not to store duplicate nodes first to avoid infinite recursion + nodeNew = true + } + ok := child.addKey(value, timestamp) + if nodeNew && ok { + v.SubNodes[nodetype] = child + } +} + +func (n *WhoisNode) MarshalJSON() ([]byte, error) { + return json.Marshal(n.Values) +} + +func (n *WhoisNode) addKey(value string, timestamp time.Time) bool { + storedValue := n.rootNode.GetValueData(value, n.nodetype) + if storedValue != nil { + if storedValue.Timestamp != nil && storedValue.Timestamp.Before(timestamp) { + storedValue.Timestamp = ×tamp + } + return false + } + n.Values[value] = WhoisValue{Timestamp: ×tamp, SubNodes: map[string]*WhoisNode{}} + return true +} + +func (n *WhoisNode) GetValueData(search, searchType string) *WhoisValue { + for value, valueData := range n.Values { + if n.nodetype == searchType && search == value { + return &valueData + } + for _, nextNode := range valueData.SubNodes { + if nextNode != nil { + if valueData := nextNode.GetValueData(search, searchType); valueData != nil { + return valueData + } + } + } + } + return nil +} + +func (n *WhoisNode) filter() { + switch n.nodetype { + case "ip": + n.filterDBUsingIP() + case "uuid": + n.FilterDBUsingUUID() + case "nodename": + n.FilterDBUsingNodeName() + } + + for _, valueData := range n.Values { + for _, nextNode := range valueData.SubNodes { + if nextNode != nil { + nextNode.filter() + } + } + } +} + +func (n *WhoisNode) filterDBUsingIP() { + for ip, valueData := range n.Values { + for hash, ip2 := range db.HashToIP { + if ip == ip2.Value { + valueData.AddChildKey(n, "uuid", hash, ip2.Timestamp) + } + } + nodenames, ok := db.IPToNodeNames[ip] + if ok { + for _, nodename := range nodenames { + valueData.AddChildKey(n, "nodename", nodename.Value, nodename.Timestamp) + } + } + } + + return +} + +func (n *WhoisNode) FilterDBUsingUUID() { + for uuid, valueData := range n.Values { + nodenames, ok := db.HashToNodeNames[uuid] + if ok { + for _, nodename := range nodenames { + valueData.AddChildKey(n, "nodename", nodename.Value, nodename.Timestamp) + } + } + ip, ok := db.HashToIP[uuid] + if ok { + valueData.AddChildKey(n, "ip", ip.Value, ip.Timestamp) + } + } + + return +} + +func (n *WhoisNode) FilterDBUsingNodeName() { + for nodename, valueData := range n.Values { + for uuid, nodenames2 := range db.HashToNodeNames { + for _, nodename2 := range nodenames2 { + if nodename == nodename2.Value { + valueData.AddChildKey(n, "uuid", uuid, nodename2.Timestamp) + } + } + } + for ip, nodenames2 := range db.IPToNodeNames { + for _, nodename2 := range nodenames2 { + if nodename == nodename2.Value { + valueData.AddChildKey(n, "ip", ip, nodename2.Timestamp) + } + } + } + } + + return +} diff --git a/src/go/pt-galera-log-explainer/types/nodeinfo.go b/src/go/pt-galera-log-explainer/types/nodeinfo.go index d1229050..260c44b5 100644 --- a/src/go/pt-galera-log-explainer/types/nodeinfo.go +++ b/src/go/pt-galera-log-explainer/types/nodeinfo.go @@ -1,12 +1,8 @@ package types -// NodeInfo is mainly used by "whois" subcommand -// This is to display its result -// As it's the base work for "sed" subcommand, it's in types package -type NodeInfo struct { +type WhoisOutput struct { Input string `json:"input"` IPs []string `json:"IPs"` NodeNames []string `json:"nodeNames"` - Hostname string `json:"hostname"` NodeUUIDs []string `json:"nodeUUIDs:"` } diff --git a/src/go/pt-galera-log-explainer/whois.go b/src/go/pt-galera-log-explainer/whois.go index 35349b7c..d5023490 100644 --- a/src/go/pt-galera-log-explainer/whois.go +++ b/src/go/pt-galera-log-explainer/whois.go @@ -1,94 +1,70 @@ package main -/* +import ( + "encoding/json" + "fmt" + + "github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/regex" + "github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/translate" + "github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/utils" + "github.com/pkg/errors" + "github.com/rs/zerolog/log" +) + type whois struct { - Search string `arg:"" name:"search" help:"the identifier (node name, ip, uuid, hash) to search"` - Paths []string `arg:"" name:"paths" help:"paths of the log to use"` + Search string `arg:"" name:"search" help:"the identifier (node name, ip, uuid) to search"` + SearchType string `name:"type" help:"what kind of information is the input (node name, ip, uuid). Auto-detected when possible." enum:"nodename,ip,uuid,auto" default:"auto"` + Paths []string `arg:"" name:"paths" help:"paths of the log to use"` } func (w *whois) Help() string { return `Take any type of info pasted from error logs and find out about it. -It will list known node name(s), IP(s), hostname(s), and other known node's UUIDs. +It will list known node name(s), IP(s), and other known node's UUIDs. + +Regarding UUIDs (wsrep_gcomm_uuid), different format can be found in logs depending on versions : +- UUID, example: ac0f3910-9790-486c-afd4-845d0ae95692 +- short UUID, with only 1st and 4st part: ac0f3910-afd4 +- shortest UUID, with only the 1st part: ac0f3910 ` } func (w *whois) Run() error { - toCheck := regex.AllRegexes() - timeline, err := timelineFromPaths(CLI.Whois.Paths, toCheck) + if w.SearchType == "auto" { + if regex.IsNodeUUID(w.Search) { + w.Search = utils.UUIDToShortUUID(w.Search) + w.SearchType = "uuid" + } else if regex.IsNodeIP(w.Search) { + w.SearchType = "ip" + } else if len(w.Search) != 8 { // at this point it's only a doubt between names and legacy node uuid, where only the first part of the uuid was shown in log + w.SearchType = "nodename" + } else { + log.Info().Msg("input information's type is ambiguous, scanning files to discover the type. You can also provide --type to avoid auto-detection") + } + } + + _, err := timelineFromPaths(CLI.Whois.Paths, regex.AllRegexes()) if err != nil { return errors.Wrap(err, "found nothing to translate") } - ctxs := timeline.GetLatestContextsByNodes() - ni := whoIs(ctxs, CLI.Whois.Search) + if w.SearchType == "auto" { + if translate.IsNodeUUIDKnown(w.Search) { + w.SearchType = "uuid" + } else if translate.IsNodeNameKnown(w.Search) { + w.SearchType = "nodename" + } else { + return errors.New("could not detect the type of input. Try to provide --type. It may means the info is unknown") + } + } - json, err := json.MarshalIndent(ni, "", "\t") + log.Debug().Str("searchType", w.SearchType).Msg("whois searchType") + out := translate.Whois(w.Search, w.SearchType) + + json, err := json.MarshalIndent(out, "", "\t") if err != nil { return err } fmt.Println(string(json)) return nil } - -func whoIs(ctxs map[string]types.LogCtx, search string) types.NodeInfo { - ni := types.NodeInfo{Input: search} - if regex.IsNodeUUID(search) { - search = utils.UUIDToShortUUID(search) - } - var ( - ips []string - hashes []string - nodenames []string - ) - for _, ctx := range ctxs { - if utils.SliceContains(ctx.OwnNames, search) || utils.SliceContains(ctx.OwnHashes, search) || utils.SliceContains(ctx.OwnIPs, search) { - ni.NodeNames = ctx.OwnNames - ni.NodeUUIDs = ctx.OwnHashes - ni.IPs = ctx.OwnIPs - ni.Hostname = ctx.OwnHostname() - } - - if nodename, ok := ctx.HashToNodeName[search]; ok { - nodenames = utils.SliceMergeDeduplicate(nodenames, []string{nodename}) - hashes = utils.SliceMergeDeduplicate(hashes, []string{search}) - } - - if ip, ok := ctx.HashToIP[search]; ok { - ips = utils.SliceMergeDeduplicate(ips, []string{ip}) - hashes = utils.SliceMergeDeduplicate(hashes, []string{search}) - - } else if nodename, ok := ctx.IPToNodeName[search]; ok { - nodenames = utils.SliceMergeDeduplicate(nodenames, []string{nodename}) - ips = utils.SliceMergeDeduplicate(ips, []string{search}) - - } else if utils.SliceContains(ctx.AllNodeNames(), search) { - nodenames = utils.SliceMergeDeduplicate(nodenames, []string{search}) - } - - for _, nodename := range nodenames { - hashes = utils.SliceMergeDeduplicate(hashes, ctx.HashesFromNodeName(nodename)) - ips = utils.SliceMergeDeduplicate(ips, ctx.IPsFromNodeName(nodename)) - } - - for _, ip := range ips { - hashes = utils.SliceMergeDeduplicate(hashes, ctx.HashesFromIP(ip)) - nodename, ok := ctx.IPToNodeName[ip] - if ok { - nodenames = utils.SliceMergeDeduplicate(nodenames, []string{nodename}) - } - } - for _, hash := range hashes { - nodename, ok := ctx.HashToNodeName[hash] - if ok { - nodenames = utils.SliceMergeDeduplicate(nodenames, []string{nodename}) - } - } - } - ni.NodeNames = nodenames - ni.NodeUUIDs = hashes - ni.IPs = ips - return ni - return types.NodeInfo{} -} -*/