pt-galera-log-explainer: add custom regexes parameter

That way anybody can handle non-galera specific logs in a single view

tests/expected/operator_auto_ambiguous_ips_list_all_no_color has been
updated, it was out-of-sync due to 2 previous pull requests that were
made in parallel

docs/pt-galera-log-explainer.rst

@@ -150,7 +150,7 @@ Available flags
     Detection is done using a prefix regex.
 
 ``--exclude-regexes``
-    Remove regexes from analysis. Use 'pt-galera-log-explainer regex-list | jq .' to have the list
+    Remove regexes from analysis. Use ``pt-galera-log-explainer regex-list | jq .`` to have the list
     
 ``--grep-cmd``
     grep v3 binary command path. For Darwin systems, it could need to be set to ``ggrep``
@@ -159,6 +159,11 @@ Available flags
 ``--version``
     Show version and exit.
 
+``--custom-regexes``
+    Add custom regexes, printed in magenta. Format: (golang regex string)=[optional static message to display].
+    If the static message is left empty, the captured string will be printed instead. Custom regexes are separated using semi-colon.
+    Example: ``--custom-regexes="Page cleaner took [0-9]*ms to flush [0-9]* pages=;doesn't recommend.*pxc_strict_mode=unsafe query used"``
+
 
 Example outputs
 ===============

src/go/pt-galera-log-explainer/README.rst

@@ -150,7 +150,7 @@ Available flags
     Detection is done using a prefix regex.
 
 ``--exclude-regexes``
-    Remove regexes from analysis. Use 'pt-galera-log-explainer regex-list | jq .' to have the list
+    Remove regexes from analysis. Use ``pt-galera-log-explainer regex-list | jq .`` to have the list
     
 ``--grep-cmd``
     grep v3 binary command path. For Darwin systems, it could need to be set to ``ggrep``
@@ -159,6 +159,11 @@ Available flags
 ``--version``
     Show version and exit.
 
+``--custom-regexes``
+    Add custom regexes, printed in magenta. Format: (golang regex string)=[optional static message to display].
+    If the static message is left empty, the captured string will be printed instead. Custom regexes are separated using semi-colon.
+    Example: ``--custom-regexes="Page cleaner took [0-9]*ms to flush [0-9]* pages=;doesn't recommend.*pxc_strict_mode=unsafe query used"``
+
 
 Example outputs
 ===============

src/go/pt-galera-log-explainer/list.go

@@ -87,5 +87,6 @@ func (l *list) regexesToUse() types.RegexMap {
 		regex.SetVerbosity(types.DebugMySQL, regex.EventsMap)
 		toCheck.Merge(regex.EventsMap)
 	}
+	toCheck.Merge(regex.CustomMap)
 	return toCheck
 }

src/go/pt-galera-log-explainer/main.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/alecthomas/kong"
+	"github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/regex"
 	"github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/translate"
 	"github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/types"
 	"github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/utils"
@@ -48,6 +49,8 @@ var CLI struct {
 	Version kong.VersionFlag
 
 	GrepCmd string `help:"'grep' command path. Could need to be set to 'ggrep' for darwin systems" default:"grep"`
+
+	CustomRegexes map[string]string `help:"Add custom regexes, printed in magenta. Format: (golang regex string)=[optional static message to display]. If the static message is left empty, the captured string will be printed instead. Custom regexes are separated using semi-colon. Example: --custom-regexes=\"Page cleaner took [0-9]*ms to flush [0-9]* pages=;doesn't recommend.*pxc_strict_mode=unsafe query used\""`
 }
 
 func main() {
@@ -70,6 +73,9 @@ func main() {
 
 	utils.SkipColor = CLI.NoColor
 
+	err := regex.AddCustomRegexes(CLI.CustomRegexes)
+	kongcli.FatalIfErrorf(err)
+
 	for _, path := range kongcli.Path {
 		if path.Positional != nil && path.Positional.Name == "paths" {
 			paths, ok := path.Positional.Target.Interface().([]string)
@@ -82,6 +88,6 @@ func main() {
 
 	translate.AssumeIPStable = !CLI.PxcOperator
 
-	err := kongcli.Run()
+	err = kongcli.Run()
 	kongcli.FatalIfErrorf(err)
 }

src/go/pt-galera-log-explainer/main_test.go

@@ -121,6 +121,22 @@ func TestMain(t *testing.T) {
 			cmd:  []string{"list", "--all", "--no-color"},
 			path: "tests/logs/conflict/*",
 		},
+
+		{
+			name: "merge_rotated_daily_list_all_custom_regex_dynamic_output_no_color",
+			cmd:  []string{"list", "--all", "--custom-regexes=Page cleaner took [0-9]*ms to flush 2000=", "--no-color"},
+			path: "tests/logs/merge_rotated_daily/node1.20230315.log",
+		},
+		{
+			name: "merge_rotated_daily_list_all_custom_regex_static_output_no_color",
+			cmd:  []string{"list", "--all", "--custom-regexes=Page cleaner took=some custom static msg", "--no-color"},
+			path: "tests/logs/merge_rotated_daily/node1.20230315.log",
+		},
+		{
+			name: "merge_rotated_daily_list_all_multiple_custom_regex_dynamic_output_no_color",
+			cmd:  []string{"list", "--all", "--custom-regexes=Page cleaner took [0-9]*ms to flush 2000=;use of .*pxc_strict_mode=", "--no-color"},
+			path: "tests/logs/merge_rotated_daily/node1.20230315.log",
+		},
 	}
 
 TESTS:

src/go/pt-galera-log-explainer/regex/custom.go

@@ -0,0 +1,43 @@
+package regex
+
+import (
+	"regexp"
+	"time"
+
+	"github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/types"
+	"github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/utils"
+	"github.com/pkg/errors"
+)
+
+var CustomMap = types.RegexMap{}
+
+func AddCustomRegexes(regexes map[string]string) error {
+	for regexstring, output := range regexes {
+		r, err := regexp.Compile(regexstring)
+		if err != nil {
+			return errors.Wrap(err, "failed to add custom regex")
+		}
+
+		lr := &types.LogRegex{Regex: r, Type: types.CustomRegexType}
+
+		if output == "" {
+			// capture and print everything that matched, instead of a static message
+			lr.InternalRegex, err = regexp.Compile("(?P<all>" + regexstring + ")")
+			if err != nil {
+				return errors.Wrap(err, "failed to add custom regex: failed to generate dynamic output")
+			}
+
+			lr.Handler = func(submatch map[string]string, ctx types.LogCtx, _ string, _ time.Time) (types.LogCtx, types.LogDisplayer) {
+				return ctx, types.SimpleDisplayer(utils.Paint(utils.MagentaText, submatch["all"]))
+			}
+
+		} else {
+			lr.Handler = func(_ map[string]string, ctx types.LogCtx, _ string, _ time.Time) (types.LogCtx, types.LogDisplayer) {
+				return ctx, types.SimpleDisplayer(utils.Paint(utils.MagentaText, output))
+			}
+		}
+
+		CustomMap[regexstring] = lr
+	}
+	return nil
+}

src/go/pt-galera-log-explainer/regex/regex.go

@@ -34,7 +34,7 @@ func SetVerbosity(verbosity types.Verbosity, regexes types.RegexMap) {
 }
 
 func AllRegexes() types.RegexMap {
-	IdentsMap.Merge(ViewsMap).Merge(SSTMap).Merge(EventsMap).Merge(StatesMap).Merge(ApplicativeMap)
+	IdentsMap.Merge(ViewsMap).Merge(SSTMap).Merge(EventsMap).Merge(StatesMap).Merge(ApplicativeMap).Merge(CustomMap)
 	return IdentsMap
 }
 

src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_custom_regex_dynamic_output_no_color

@@ -0,0 +1,22 @@
+identifier                         node1                                               
+current path                       tests/logs/merge_rotated_daily/node1.20230315.log   
+last known ip                                                                          
+last known name                    node1                                               
+mysql version                                                                          
+                                                                                       
+2023-03-15T10:12:43.720576+02:00   Page cleaner took 5165ms to flush 2000              
+2023-03-15T10:12:51.514442+02:00   Page cleaner took 5793ms to flush 2000              
+2023-03-15T10:13:22.185343+02:00   Page cleaner took 4063ms to flush 2000              
+2023-03-15T20:10:57.784904+02:00   node2 joined                                        
+2023-03-15T20:10:57.785568+02:00   node3 left                                          
+2023-03-15T20:10:57.791959+02:00   node3 left                                          
+2023-03-15T20:10:57.797221+02:00   PRIMARY(n=2)                                        
+2023-03-15T20:20:12.714291+02:00   node2 joined                                        
+2023-03-15T20:20:12.714331+02:00   node3 joined                                        
+2023-03-15T20:20:13.776977+02:00   PRIMARY(n=3)                                        
+2023-03-15T20:20:14.839684+02:00   local node will resync node3                        
+2023-03-15T20:20:14.839723+02:00   SYNCED -> DONOR                                     
+2023-03-15T20:20:15.799020+02:00   IST will be used                                    
+2023-03-15T20:20:16.850525+02:00   finished sending IST to node3                       
+2023-03-15T20:20:16.850549+02:00   DESYNCED -> JOINED                                  
+2023-03-15T20:20:16.865312+02:00   JOINED -> SYNCED                                    

src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_custom_regex_static_output_no_color

@@ -0,0 +1,21 @@
+identifier                         node1                                               
+current path                       tests/logs/merge_rotated_daily/node1.20230315.log   
+last known ip                                                                          
+last known name                    node1                                               
+mysql version                                                                          
+                                                                                       
+2023-03-15T10:12:43.720576+02:00   (repeated x2)some custom static msg                 
+2023-03-15T10:36:44.899671+02:00   some custom static msg                              
+2023-03-15T20:10:57.784904+02:00   node2 joined                                        
+2023-03-15T20:10:57.785568+02:00   node3 left                                          
+2023-03-15T20:10:57.791959+02:00   node3 left                                          
+2023-03-15T20:10:57.797221+02:00   PRIMARY(n=2)                                        
+2023-03-15T20:20:12.714291+02:00   node2 joined                                        
+2023-03-15T20:20:12.714331+02:00   node3 joined                                        
+2023-03-15T20:20:13.776977+02:00   PRIMARY(n=3)                                        
+2023-03-15T20:20:14.839684+02:00   local node will resync node3                        
+2023-03-15T20:20:14.839723+02:00   SYNCED -> DONOR                                     
+2023-03-15T20:20:15.799020+02:00   IST will be used                                    
+2023-03-15T20:20:16.850525+02:00   finished sending IST to node3                       
+2023-03-15T20:20:16.850549+02:00   DESYNCED -> JOINED                                  
+2023-03-15T20:20:16.865312+02:00   JOINED -> SYNCED                                    

src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_multiple_custom_regex_dynamic_output_no_color

@@ -0,0 +1,48 @@
+identifier                         node1                                               
+current path                       tests/logs/merge_rotated_daily/node1.20230315.log   
+last known ip                                                                          
+last known name                    node1                                               
+mysql version                                                                          
+                                                                                       
+2023-03-15T05:08:37.642287+02:00   use of GET_LOCK with pxc_strict_mode                
+2023-03-15T05:08:37.655777+02:00   use of RELEASE_LOCK with pxc_strict_mode            
+2023-03-15T05:12:16.145677+02:00   use of GET_LOCK with pxc_strict_mode                
+2023-03-15T05:12:16.158686+02:00   use of RELEASE_LOCK with pxc_strict_mode            
+2023-03-15T05:18:21.053512+02:00   use of GET_LOCK with pxc_strict_mode                
+2023-03-15T05:18:21.076291+02:00   use of RELEASE_LOCK with pxc_strict_mode            
+2023-03-15T05:37:25.194770+02:00   use of GET_LOCK with pxc_strict_mode                
+2023-03-15T05:37:25.243590+02:00   use of RELEASE_LOCK with pxc_strict_mode            
+2023-03-15T06:58:53.762488+02:00   use of GET_LOCK with pxc_strict_mode                
+2023-03-15T06:58:53.843783+02:00   use of RELEASE_LOCK with pxc_strict_mode            
+2023-03-15T09:12:04.577365+02:00   use of GET_LOCK with pxc_strict_mode                
+2023-03-15T09:12:05.290288+02:00   use of RELEASE_LOCK with pxc_strict_mode            
+2023-03-15T09:12:05.296372+02:00   use of GET_LOCK with pxc_strict_mode                
+2023-03-15T09:12:05.310008+02:00   use of RELEASE_LOCK with pxc_strict_mode            
+2023-03-15T10:12:43.720576+02:00   Page cleaner took 5165ms to flush 2000              
+2023-03-15T10:12:51.514442+02:00   Page cleaner took 5793ms to flush 2000              
+2023-03-15T10:13:22.185343+02:00   Page cleaner took 4063ms to flush 2000              
+2023-03-15T13:13:34.584680+02:00   use of GET_LOCK with pxc_strict_mode                
+2023-03-15T13:13:34.597810+02:00   use of RELEASE_LOCK with pxc_strict_mode            
+2023-03-15T13:51:45.383215+02:00   use of GET_LOCK with pxc_strict_mode                
+2023-03-15T13:51:45.396164+02:00   use of RELEASE_LOCK with pxc_strict_mode            
+2023-03-15T15:44:04.523882+02:00   use of GET_LOCK with pxc_strict_mode                
+2023-03-15T15:44:04.534949+02:00   use of RELEASE_LOCK with pxc_strict_mode            
+2023-03-15T16:05:37.269541+02:00   use of GET_LOCK with pxc_strict_mode                
+2023-03-15T16:05:37.295035+02:00   use of RELEASE_LOCK with pxc_strict_mode            
+2023-03-15T20:10:57.784904+02:00   node2 joined                                        
+2023-03-15T20:10:57.785568+02:00   node3 left                                          
+2023-03-15T20:10:57.791959+02:00   node3 left                                          
+2023-03-15T20:10:57.797221+02:00   PRIMARY(n=2)                                        
+2023-03-15T20:18:57.060660+02:00   use of GET_LOCK with pxc_strict_mode                
+2023-03-15T20:18:57.081839+02:00   use of RELEASE_LOCK with pxc_strict_mode            
+2023-03-15T20:20:12.714291+02:00   node2 joined                                        
+2023-03-15T20:20:12.714331+02:00   node3 joined                                        
+2023-03-15T20:20:13.776977+02:00   PRIMARY(n=3)                                        
+2023-03-15T20:20:14.839684+02:00   local node will resync node3                        
+2023-03-15T20:20:14.839723+02:00   SYNCED -> DONOR                                     
+2023-03-15T20:20:15.799020+02:00   IST will be used                                    
+2023-03-15T20:20:16.850525+02:00   finished sending IST to node3                       
+2023-03-15T20:20:16.850549+02:00   DESYNCED -> JOINED                                  
+2023-03-15T20:20:16.865312+02:00   JOINED -> SYNCED                                    
+2023-03-15T20:39:36.865219+02:00   use of GET_LOCK with pxc_strict_mode                
+2023-03-15T20:39:36.946588+02:00   use of RELEASE_LOCK with pxc_strict_mode            

src/go/pt-galera-log-explainer/tests/expected/operator_auto_ambiguous_ips_list_all_no_color

@@ -505,6 +505,7 @@ mysql version                 8.0.31                                        8.0.
 2023-05-16T07:51:58.379294Z   |                                             recovering gcache                             |                                             
 2023-05-16T07:52:26.431252Z   |                                             CLOSED -> OPEN                                |                                             
 2023-05-16T07:52:26.431427Z   |                                             NON-PRIMARY(n=1)                              |                                             
+2023-05-16T07:52:26.931116Z   |                                             inactive check more than 1.5s (3.50171s)      |                                             
 2023-05-16T10:03:18.655740Z   starting(8.0.31)                              |                                             |                                             
 2023-05-16T10:03:18.658268Z   started(cluster)                              |                                             |                                             
 2023-05-16T10:03:18.659239Z   no grastate.dat file                          |                                             |                                             

src/go/pt-galera-log-explainer/types/regex.go

@@ -71,6 +71,7 @@ var (
 	StatesRegexType      RegexType = "states"
 	PXCOperatorRegexType RegexType = "pxc-operator"
 	ApplicativeRegexType RegexType = "applicative"
+	CustomRegexType      RegexType = "custom"
 )
 
 type RegexMap map[string]*LogRegex