From bebd652600c93ba72060d800baf0f159b099beb1 Mon Sep 17 00:00:00 2001 From: Yoann La Cancellera Date: Mon, 19 Feb 2024 20:08:29 +0100 Subject: [PATCH] pt-galera-log-explainer: add custom regexes parameter That way anybody can handle non-galera specific logs in a single view tests/expected/operator_auto_ambiguous_ips_list_all_no_color has been updated, it was out-of-sync due to 2 previous pull requests that were made in parallel --- docs/pt-galera-log-explainer.rst | 7 ++- src/go/pt-galera-log-explainer/README.rst | 7 ++- src/go/pt-galera-log-explainer/list.go | 1 + src/go/pt-galera-log-explainer/main.go | 8 +++- src/go/pt-galera-log-explainer/main_test.go | 16 +++++++ .../pt-galera-log-explainer/regex/custom.go | 43 +++++++++++++++++ src/go/pt-galera-log-explainer/regex/regex.go | 2 +- ...t_all_custom_regex_dynamic_output_no_color | 22 +++++++++ ...st_all_custom_regex_static_output_no_color | 21 ++++++++ ...tiple_custom_regex_dynamic_output_no_color | 48 +++++++++++++++++++ ...rator_auto_ambiguous_ips_list_all_no_color | 1 + src/go/pt-galera-log-explainer/types/regex.go | 1 + 12 files changed, 173 insertions(+), 4 deletions(-) create mode 100644 src/go/pt-galera-log-explainer/regex/custom.go create mode 100644 src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_custom_regex_dynamic_output_no_color create mode 100644 src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_custom_regex_static_output_no_color create mode 100644 src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_multiple_custom_regex_dynamic_output_no_color diff --git a/docs/pt-galera-log-explainer.rst b/docs/pt-galera-log-explainer.rst index 3fbe7e8c..52bc8161 100644 --- a/docs/pt-galera-log-explainer.rst +++ b/docs/pt-galera-log-explainer.rst @@ -150,7 +150,7 @@ Available flags Detection is done using a prefix regex. ``--exclude-regexes`` - Remove regexes from analysis. Use 'pt-galera-log-explainer regex-list | jq .' to have the list + Remove regexes from analysis. Use ``pt-galera-log-explainer regex-list | jq .`` to have the list ``--grep-cmd`` grep v3 binary command path. For Darwin systems, it could need to be set to ``ggrep`` @@ -159,6 +159,11 @@ Available flags ``--version`` Show version and exit. +``--custom-regexes`` + Add custom regexes, printed in magenta. Format: (golang regex string)=[optional static message to display]. + If the static message is left empty, the captured string will be printed instead. Custom regexes are separated using semi-colon. + Example: ``--custom-regexes="Page cleaner took [0-9]*ms to flush [0-9]* pages=;doesn't recommend.*pxc_strict_mode=unsafe query used"`` + Example outputs =============== diff --git a/src/go/pt-galera-log-explainer/README.rst b/src/go/pt-galera-log-explainer/README.rst index 3fbe7e8c..52bc8161 100644 --- a/src/go/pt-galera-log-explainer/README.rst +++ b/src/go/pt-galera-log-explainer/README.rst @@ -150,7 +150,7 @@ Available flags Detection is done using a prefix regex. ``--exclude-regexes`` - Remove regexes from analysis. Use 'pt-galera-log-explainer regex-list | jq .' to have the list + Remove regexes from analysis. Use ``pt-galera-log-explainer regex-list | jq .`` to have the list ``--grep-cmd`` grep v3 binary command path. For Darwin systems, it could need to be set to ``ggrep`` @@ -159,6 +159,11 @@ Available flags ``--version`` Show version and exit. +``--custom-regexes`` + Add custom regexes, printed in magenta. Format: (golang regex string)=[optional static message to display]. + If the static message is left empty, the captured string will be printed instead. Custom regexes are separated using semi-colon. + Example: ``--custom-regexes="Page cleaner took [0-9]*ms to flush [0-9]* pages=;doesn't recommend.*pxc_strict_mode=unsafe query used"`` + Example outputs =============== diff --git a/src/go/pt-galera-log-explainer/list.go b/src/go/pt-galera-log-explainer/list.go index 7d97241a..46670eb0 100644 --- a/src/go/pt-galera-log-explainer/list.go +++ b/src/go/pt-galera-log-explainer/list.go @@ -87,5 +87,6 @@ func (l *list) regexesToUse() types.RegexMap { regex.SetVerbosity(types.DebugMySQL, regex.EventsMap) toCheck.Merge(regex.EventsMap) } + toCheck.Merge(regex.CustomMap) return toCheck } diff --git a/src/go/pt-galera-log-explainer/main.go b/src/go/pt-galera-log-explainer/main.go index 81c1a404..7e78db1f 100644 --- a/src/go/pt-galera-log-explainer/main.go +++ b/src/go/pt-galera-log-explainer/main.go @@ -6,6 +6,7 @@ import ( "time" "github.com/alecthomas/kong" + "github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/regex" "github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/translate" "github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/types" "github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/utils" @@ -48,6 +49,8 @@ var CLI struct { Version kong.VersionFlag GrepCmd string `help:"'grep' command path. Could need to be set to 'ggrep' for darwin systems" default:"grep"` + + CustomRegexes map[string]string `help:"Add custom regexes, printed in magenta. Format: (golang regex string)=[optional static message to display]. If the static message is left empty, the captured string will be printed instead. Custom regexes are separated using semi-colon. Example: --custom-regexes=\"Page cleaner took [0-9]*ms to flush [0-9]* pages=;doesn't recommend.*pxc_strict_mode=unsafe query used\""` } func main() { @@ -70,6 +73,9 @@ func main() { utils.SkipColor = CLI.NoColor + err := regex.AddCustomRegexes(CLI.CustomRegexes) + kongcli.FatalIfErrorf(err) + for _, path := range kongcli.Path { if path.Positional != nil && path.Positional.Name == "paths" { paths, ok := path.Positional.Target.Interface().([]string) @@ -82,6 +88,6 @@ func main() { translate.AssumeIPStable = !CLI.PxcOperator - err := kongcli.Run() + err = kongcli.Run() kongcli.FatalIfErrorf(err) } diff --git a/src/go/pt-galera-log-explainer/main_test.go b/src/go/pt-galera-log-explainer/main_test.go index 27bfe34f..8ad6e2d4 100644 --- a/src/go/pt-galera-log-explainer/main_test.go +++ b/src/go/pt-galera-log-explainer/main_test.go @@ -121,6 +121,22 @@ func TestMain(t *testing.T) { cmd: []string{"list", "--all", "--no-color"}, path: "tests/logs/conflict/*", }, + + { + name: "merge_rotated_daily_list_all_custom_regex_dynamic_output_no_color", + cmd: []string{"list", "--all", "--custom-regexes=Page cleaner took [0-9]*ms to flush 2000=", "--no-color"}, + path: "tests/logs/merge_rotated_daily/node1.20230315.log", + }, + { + name: "merge_rotated_daily_list_all_custom_regex_static_output_no_color", + cmd: []string{"list", "--all", "--custom-regexes=Page cleaner took=some custom static msg", "--no-color"}, + path: "tests/logs/merge_rotated_daily/node1.20230315.log", + }, + { + name: "merge_rotated_daily_list_all_multiple_custom_regex_dynamic_output_no_color", + cmd: []string{"list", "--all", "--custom-regexes=Page cleaner took [0-9]*ms to flush 2000=;use of .*pxc_strict_mode=", "--no-color"}, + path: "tests/logs/merge_rotated_daily/node1.20230315.log", + }, } TESTS: diff --git a/src/go/pt-galera-log-explainer/regex/custom.go b/src/go/pt-galera-log-explainer/regex/custom.go new file mode 100644 index 00000000..82758343 --- /dev/null +++ b/src/go/pt-galera-log-explainer/regex/custom.go @@ -0,0 +1,43 @@ +package regex + +import ( + "regexp" + "time" + + "github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/types" + "github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/utils" + "github.com/pkg/errors" +) + +var CustomMap = types.RegexMap{} + +func AddCustomRegexes(regexes map[string]string) error { + for regexstring, output := range regexes { + r, err := regexp.Compile(regexstring) + if err != nil { + return errors.Wrap(err, "failed to add custom regex") + } + + lr := &types.LogRegex{Regex: r, Type: types.CustomRegexType} + + if output == "" { + // capture and print everything that matched, instead of a static message + lr.InternalRegex, err = regexp.Compile("(?P" + regexstring + ")") + if err != nil { + return errors.Wrap(err, "failed to add custom regex: failed to generate dynamic output") + } + + lr.Handler = func(submatch map[string]string, ctx types.LogCtx, _ string, _ time.Time) (types.LogCtx, types.LogDisplayer) { + return ctx, types.SimpleDisplayer(utils.Paint(utils.MagentaText, submatch["all"])) + } + + } else { + lr.Handler = func(_ map[string]string, ctx types.LogCtx, _ string, _ time.Time) (types.LogCtx, types.LogDisplayer) { + return ctx, types.SimpleDisplayer(utils.Paint(utils.MagentaText, output)) + } + } + + CustomMap[regexstring] = lr + } + return nil +} diff --git a/src/go/pt-galera-log-explainer/regex/regex.go b/src/go/pt-galera-log-explainer/regex/regex.go index 99962d23..44f8210c 100644 --- a/src/go/pt-galera-log-explainer/regex/regex.go +++ b/src/go/pt-galera-log-explainer/regex/regex.go @@ -34,7 +34,7 @@ func SetVerbosity(verbosity types.Verbosity, regexes types.RegexMap) { } func AllRegexes() types.RegexMap { - IdentsMap.Merge(ViewsMap).Merge(SSTMap).Merge(EventsMap).Merge(StatesMap).Merge(ApplicativeMap) + IdentsMap.Merge(ViewsMap).Merge(SSTMap).Merge(EventsMap).Merge(StatesMap).Merge(ApplicativeMap).Merge(CustomMap) return IdentsMap } diff --git a/src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_custom_regex_dynamic_output_no_color b/src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_custom_regex_dynamic_output_no_color new file mode 100644 index 00000000..4f57828d --- /dev/null +++ b/src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_custom_regex_dynamic_output_no_color @@ -0,0 +1,22 @@ +identifier node1 +current path tests/logs/merge_rotated_daily/node1.20230315.log +last known ip +last known name node1 +mysql version + +2023-03-15T10:12:43.720576+02:00 Page cleaner took 5165ms to flush 2000 +2023-03-15T10:12:51.514442+02:00 Page cleaner took 5793ms to flush 2000 +2023-03-15T10:13:22.185343+02:00 Page cleaner took 4063ms to flush 2000 +2023-03-15T20:10:57.784904+02:00 node2 joined +2023-03-15T20:10:57.785568+02:00 node3 left +2023-03-15T20:10:57.791959+02:00 node3 left +2023-03-15T20:10:57.797221+02:00 PRIMARY(n=2) +2023-03-15T20:20:12.714291+02:00 node2 joined +2023-03-15T20:20:12.714331+02:00 node3 joined +2023-03-15T20:20:13.776977+02:00 PRIMARY(n=3) +2023-03-15T20:20:14.839684+02:00 local node will resync node3 +2023-03-15T20:20:14.839723+02:00 SYNCED -> DONOR +2023-03-15T20:20:15.799020+02:00 IST will be used +2023-03-15T20:20:16.850525+02:00 finished sending IST to node3 +2023-03-15T20:20:16.850549+02:00 DESYNCED -> JOINED +2023-03-15T20:20:16.865312+02:00 JOINED -> SYNCED diff --git a/src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_custom_regex_static_output_no_color b/src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_custom_regex_static_output_no_color new file mode 100644 index 00000000..bda1ca02 --- /dev/null +++ b/src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_custom_regex_static_output_no_color @@ -0,0 +1,21 @@ +identifier node1 +current path tests/logs/merge_rotated_daily/node1.20230315.log +last known ip +last known name node1 +mysql version + +2023-03-15T10:12:43.720576+02:00 (repeated x2)some custom static msg +2023-03-15T10:36:44.899671+02:00 some custom static msg +2023-03-15T20:10:57.784904+02:00 node2 joined +2023-03-15T20:10:57.785568+02:00 node3 left +2023-03-15T20:10:57.791959+02:00 node3 left +2023-03-15T20:10:57.797221+02:00 PRIMARY(n=2) +2023-03-15T20:20:12.714291+02:00 node2 joined +2023-03-15T20:20:12.714331+02:00 node3 joined +2023-03-15T20:20:13.776977+02:00 PRIMARY(n=3) +2023-03-15T20:20:14.839684+02:00 local node will resync node3 +2023-03-15T20:20:14.839723+02:00 SYNCED -> DONOR +2023-03-15T20:20:15.799020+02:00 IST will be used +2023-03-15T20:20:16.850525+02:00 finished sending IST to node3 +2023-03-15T20:20:16.850549+02:00 DESYNCED -> JOINED +2023-03-15T20:20:16.865312+02:00 JOINED -> SYNCED diff --git a/src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_multiple_custom_regex_dynamic_output_no_color b/src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_multiple_custom_regex_dynamic_output_no_color new file mode 100644 index 00000000..a57bf91c --- /dev/null +++ b/src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_multiple_custom_regex_dynamic_output_no_color @@ -0,0 +1,48 @@ +identifier node1 +current path tests/logs/merge_rotated_daily/node1.20230315.log +last known ip +last known name node1 +mysql version + +2023-03-15T05:08:37.642287+02:00 use of GET_LOCK with pxc_strict_mode +2023-03-15T05:08:37.655777+02:00 use of RELEASE_LOCK with pxc_strict_mode +2023-03-15T05:12:16.145677+02:00 use of GET_LOCK with pxc_strict_mode +2023-03-15T05:12:16.158686+02:00 use of RELEASE_LOCK with pxc_strict_mode +2023-03-15T05:18:21.053512+02:00 use of GET_LOCK with pxc_strict_mode +2023-03-15T05:18:21.076291+02:00 use of RELEASE_LOCK with pxc_strict_mode +2023-03-15T05:37:25.194770+02:00 use of GET_LOCK with pxc_strict_mode +2023-03-15T05:37:25.243590+02:00 use of RELEASE_LOCK with pxc_strict_mode +2023-03-15T06:58:53.762488+02:00 use of GET_LOCK with pxc_strict_mode +2023-03-15T06:58:53.843783+02:00 use of RELEASE_LOCK with pxc_strict_mode +2023-03-15T09:12:04.577365+02:00 use of GET_LOCK with pxc_strict_mode +2023-03-15T09:12:05.290288+02:00 use of RELEASE_LOCK with pxc_strict_mode +2023-03-15T09:12:05.296372+02:00 use of GET_LOCK with pxc_strict_mode +2023-03-15T09:12:05.310008+02:00 use of RELEASE_LOCK with pxc_strict_mode +2023-03-15T10:12:43.720576+02:00 Page cleaner took 5165ms to flush 2000 +2023-03-15T10:12:51.514442+02:00 Page cleaner took 5793ms to flush 2000 +2023-03-15T10:13:22.185343+02:00 Page cleaner took 4063ms to flush 2000 +2023-03-15T13:13:34.584680+02:00 use of GET_LOCK with pxc_strict_mode +2023-03-15T13:13:34.597810+02:00 use of RELEASE_LOCK with pxc_strict_mode +2023-03-15T13:51:45.383215+02:00 use of GET_LOCK with pxc_strict_mode +2023-03-15T13:51:45.396164+02:00 use of RELEASE_LOCK with pxc_strict_mode +2023-03-15T15:44:04.523882+02:00 use of GET_LOCK with pxc_strict_mode +2023-03-15T15:44:04.534949+02:00 use of RELEASE_LOCK with pxc_strict_mode +2023-03-15T16:05:37.269541+02:00 use of GET_LOCK with pxc_strict_mode +2023-03-15T16:05:37.295035+02:00 use of RELEASE_LOCK with pxc_strict_mode +2023-03-15T20:10:57.784904+02:00 node2 joined +2023-03-15T20:10:57.785568+02:00 node3 left +2023-03-15T20:10:57.791959+02:00 node3 left +2023-03-15T20:10:57.797221+02:00 PRIMARY(n=2) +2023-03-15T20:18:57.060660+02:00 use of GET_LOCK with pxc_strict_mode +2023-03-15T20:18:57.081839+02:00 use of RELEASE_LOCK with pxc_strict_mode +2023-03-15T20:20:12.714291+02:00 node2 joined +2023-03-15T20:20:12.714331+02:00 node3 joined +2023-03-15T20:20:13.776977+02:00 PRIMARY(n=3) +2023-03-15T20:20:14.839684+02:00 local node will resync node3 +2023-03-15T20:20:14.839723+02:00 SYNCED -> DONOR +2023-03-15T20:20:15.799020+02:00 IST will be used +2023-03-15T20:20:16.850525+02:00 finished sending IST to node3 +2023-03-15T20:20:16.850549+02:00 DESYNCED -> JOINED +2023-03-15T20:20:16.865312+02:00 JOINED -> SYNCED +2023-03-15T20:39:36.865219+02:00 use of GET_LOCK with pxc_strict_mode +2023-03-15T20:39:36.946588+02:00 use of RELEASE_LOCK with pxc_strict_mode diff --git a/src/go/pt-galera-log-explainer/tests/expected/operator_auto_ambiguous_ips_list_all_no_color b/src/go/pt-galera-log-explainer/tests/expected/operator_auto_ambiguous_ips_list_all_no_color index 46909326..9903f95c 100644 --- a/src/go/pt-galera-log-explainer/tests/expected/operator_auto_ambiguous_ips_list_all_no_color +++ b/src/go/pt-galera-log-explainer/tests/expected/operator_auto_ambiguous_ips_list_all_no_color @@ -505,6 +505,7 @@ mysql version 8.0.31 8.0. 2023-05-16T07:51:58.379294Z | recovering gcache | 2023-05-16T07:52:26.431252Z | CLOSED -> OPEN | 2023-05-16T07:52:26.431427Z | NON-PRIMARY(n=1) | +2023-05-16T07:52:26.931116Z | inactive check more than 1.5s (3.50171s) | 2023-05-16T10:03:18.655740Z starting(8.0.31) | | 2023-05-16T10:03:18.658268Z started(cluster) | | 2023-05-16T10:03:18.659239Z no grastate.dat file | | diff --git a/src/go/pt-galera-log-explainer/types/regex.go b/src/go/pt-galera-log-explainer/types/regex.go index 25761092..eaa014a0 100644 --- a/src/go/pt-galera-log-explainer/types/regex.go +++ b/src/go/pt-galera-log-explainer/types/regex.go @@ -71,6 +71,7 @@ var ( StatesRegexType RegexType = "states" PXCOperatorRegexType RegexType = "pxc-operator" ApplicativeRegexType RegexType = "applicative" + CustomRegexType RegexType = "custom" ) type RegexMap map[string]*LogRegex