percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-19 18:34:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

pt-galera-log-explainer: add custom regexes parameter

Browse Source 
That way anybody can handle non-galera specific logs in a single view

tests/expected/operator_auto_ambiguous_ips_list_all_no_color has been
updated, it was out-of-sync due to 2 previous pull requests that were
made in parallel
This commit is contained in:
Yoann La Cancellera
2024-02-19 20:08:29 +01:00
parent f684a11335
commit bebd652600
12 changed files with 173 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/go/pt-galera-log-explainer/README.rst
View File

@@ -150,7 +150,7 @@ Available flags
Detection is done using a prefix regex.
``--exclude-regexes``
Remove regexes from analysis. Use 'pt-galera-log-explainer regex-list | jq .' to have the list
Remove regexes from analysis. Use ``pt-galera-log-explainer regex-list | jq .`` to have the list
``--grep-cmd``
grep v3 binary command path. For Darwin systems, it could need to be set to ``ggrep``
@@ -159,6 +159,11 @@ Available flags
``--version``
Show version and exit.
``--custom-regexes``
Add custom regexes, printed in magenta. Format: (golang regex string)=[optional static message to display].
If the static message is left empty, the captured string will be printed instead. Custom regexes are separated using semi-colon.
Example: ``--custom-regexes="Page cleaner took [0-9]*ms to flush [0-9]* pages=;doesn't recommend.*pxc_strict_mode=unsafe query used"``
Example outputs
===============

1
src/go/pt-galera-log-explainer/list.go
View File

@@ -87,5 +87,6 @@ func (l *list) regexesToUse() types.RegexMap {
regex.SetVerbosity(types.DebugMySQL, regex.EventsMap)
toCheck.Merge(regex.EventsMap)
}
toCheck.Merge(regex.CustomMap)
return toCheck
}

8
src/go/pt-galera-log-explainer/main.go
View File

@@ -6,6 +6,7 @@ import (
"time"
"github.com/alecthomas/kong"
"github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/regex"
"github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/translate"
"github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/types"
"github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/utils"
@@ -48,6 +49,8 @@ var CLI struct {
Version kong.VersionFlag
GrepCmd string `help:"'grep' command path. Could need to be set to 'ggrep' for darwin systems" default:"grep"`
CustomRegexes map[string]string `help:"Add custom regexes, printed in magenta. Format: (golang regex string)=[optional static message to display]. If the static message is left empty, the captured string will be printed instead. Custom regexes are separated using semi-colon. Example: --custom-regexes=\"Page cleaner took [0-9]*ms to flush [0-9]* pages=;doesn't recommend.*pxc_strict_mode=unsafe query used\""`
}
func main() {
@@ -70,6 +73,9 @@ func main() {
utils.SkipColor = CLI.NoColor
err := regex.AddCustomRegexes(CLI.CustomRegexes)
kongcli.FatalIfErrorf(err)
for _, path := range kongcli.Path {
if path.Positional != nil && path.Positional.Name == "paths" {
paths, ok := path.Positional.Target.Interface().([]string)
@@ -82,6 +88,6 @@ func main() {
translate.AssumeIPStable = !CLI.PxcOperator
err := kongcli.Run()
err = kongcli.Run()
kongcli.FatalIfErrorf(err)
}

16
src/go/pt-galera-log-explainer/main_test.go
View File

@@ -121,6 +121,22 @@ func TestMain(t *testing.T) {
cmd: []string{"list", "--all", "--no-color"},
path: "tests/logs/conflict/*",
},
{
name: "merge_rotated_daily_list_all_custom_regex_dynamic_output_no_color",
cmd: []string{"list", "--all", "--custom-regexes=Page cleaner took [0-9]*ms to flush 2000=", "--no-color"},
path: "tests/logs/merge_rotated_daily/node1.20230315.log",
},
{
name: "merge_rotated_daily_list_all_custom_regex_static_output_no_color",
cmd: []string{"list", "--all", "--custom-regexes=Page cleaner took=some custom static msg", "--no-color"},
path: "tests/logs/merge_rotated_daily/node1.20230315.log",
},
{
name: "merge_rotated_daily_list_all_multiple_custom_regex_dynamic_output_no_color",
cmd: []string{"list", "--all", "--custom-regexes=Page cleaner took [0-9]*ms to flush 2000=;use of .*pxc_strict_mode=", "--no-color"},
path: "tests/logs/merge_rotated_daily/node1.20230315.log",
},
}
TESTS:

43
src/go/pt-galera-log-explainer/regex/custom.go Normal file
View File

@@ -0,0 +1,43 @@
package regex
import (
"regexp"
"time"
"github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/types"
"github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/utils"
"github.com/pkg/errors"
)
var CustomMap = types.RegexMap{}
func AddCustomRegexes(regexes map[string]string) error {
for regexstring, output := range regexes {
r, err := regexp.Compile(regexstring)
if err != nil {
return errors.Wrap(err, "failed to add custom regex")
}
lr := &types.LogRegex{Regex: r, Type: types.CustomRegexType}
if output == "" {
// capture and print everything that matched, instead of a static message
lr.InternalRegex, err = regexp.Compile("(?P<all>" + regexstring + ")")
if err != nil {
return errors.Wrap(err, "failed to add custom regex: failed to generate dynamic output")
}
lr.Handler = func(submatch map[string]string, ctx types.LogCtx, _ string, _ time.Time) (types.LogCtx, types.LogDisplayer) {
return ctx, types.SimpleDisplayer(utils.Paint(utils.MagentaText, submatch["all"]))
}
} else {
lr.Handler = func(_ map[string]string, ctx types.LogCtx, _ string, _ time.Time) (types.LogCtx, types.LogDisplayer) {
return ctx, types.SimpleDisplayer(utils.Paint(utils.MagentaText, output))
}
}
CustomMap[regexstring] = lr
}
return nil
}

2
src/go/pt-galera-log-explainer/regex/regex.go
View File

@@ -34,7 +34,7 @@ func SetVerbosity(verbosity types.Verbosity, regexes types.RegexMap) {
}
func AllRegexes() types.RegexMap {
IdentsMap.Merge(ViewsMap).Merge(SSTMap).Merge(EventsMap).Merge(StatesMap).Merge(ApplicativeMap)
IdentsMap.Merge(ViewsMap).Merge(SSTMap).Merge(EventsMap).Merge(StatesMap).Merge(ApplicativeMap).Merge(CustomMap)
return IdentsMap
}

22
src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_custom_regex_dynamic_output_no_color Normal file
View File

@@ -0,0 +1,22 @@
identifier node1
current path tests/logs/merge_rotated_daily/node1.20230315.log
last known ip
last known name node1
mysql version
2023-03-15T10:12:43.720576+02:00 Page cleaner took 5165ms to flush 2000
2023-03-15T10:12:51.514442+02:00 Page cleaner took 5793ms to flush 2000
2023-03-15T10:13:22.185343+02:00 Page cleaner took 4063ms to flush 2000
2023-03-15T20:10:57.784904+02:00 node2 joined
2023-03-15T20:10:57.785568+02:00 node3 left
2023-03-15T20:10:57.791959+02:00 node3 left
2023-03-15T20:10:57.797221+02:00 PRIMARY(n=2)
2023-03-15T20:20:12.714291+02:00 node2 joined
2023-03-15T20:20:12.714331+02:00 node3 joined
2023-03-15T20:20:13.776977+02:00 PRIMARY(n=3)
2023-03-15T20:20:14.839684+02:00 local node will resync node3
2023-03-15T20:20:14.839723+02:00 SYNCED -> DONOR
2023-03-15T20:20:15.799020+02:00 IST will be used
2023-03-15T20:20:16.850525+02:00 finished sending IST to node3
2023-03-15T20:20:16.850549+02:00 DESYNCED -> JOINED
2023-03-15T20:20:16.865312+02:00 JOINED -> SYNCED

21
src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_custom_regex_static_output_no_color Normal file
View File

@@ -0,0 +1,21 @@
identifier node1
current path tests/logs/merge_rotated_daily/node1.20230315.log
last known ip
last known name node1
mysql version
2023-03-15T10:12:43.720576+02:00 (repeated x2)some custom static msg
2023-03-15T10:36:44.899671+02:00 some custom static msg
2023-03-15T20:10:57.784904+02:00 node2 joined
2023-03-15T20:10:57.785568+02:00 node3 left
2023-03-15T20:10:57.791959+02:00 node3 left
2023-03-15T20:10:57.797221+02:00 PRIMARY(n=2)
2023-03-15T20:20:12.714291+02:00 node2 joined
2023-03-15T20:20:12.714331+02:00 node3 joined
2023-03-15T20:20:13.776977+02:00 PRIMARY(n=3)
2023-03-15T20:20:14.839684+02:00 local node will resync node3
2023-03-15T20:20:14.839723+02:00 SYNCED -> DONOR
2023-03-15T20:20:15.799020+02:00 IST will be used
2023-03-15T20:20:16.850525+02:00 finished sending IST to node3
2023-03-15T20:20:16.850549+02:00 DESYNCED -> JOINED
2023-03-15T20:20:16.865312+02:00 JOINED -> SYNCED

48
src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_multiple_custom_regex_dynamic_output_no_color Normal file
View File

@@ -0,0 +1,48 @@
identifier node1
current path tests/logs/merge_rotated_daily/node1.20230315.log
last known ip
last known name node1
mysql version
2023-03-15T05:08:37.642287+02:00 use of GET_LOCK with pxc_strict_mode
2023-03-15T05:08:37.655777+02:00 use of RELEASE_LOCK with pxc_strict_mode
2023-03-15T05:12:16.145677+02:00 use of GET_LOCK with pxc_strict_mode
2023-03-15T05:12:16.158686+02:00 use of RELEASE_LOCK with pxc_strict_mode
2023-03-15T05:18:21.053512+02:00 use of GET_LOCK with pxc_strict_mode
2023-03-15T05:18:21.076291+02:00 use of RELEASE_LOCK with pxc_strict_mode
2023-03-15T05:37:25.194770+02:00 use of GET_LOCK with pxc_strict_mode
2023-03-15T05:37:25.243590+02:00 use of RELEASE_LOCK with pxc_strict_mode
2023-03-15T06:58:53.762488+02:00 use of GET_LOCK with pxc_strict_mode
2023-03-15T06:58:53.843783+02:00 use of RELEASE_LOCK with pxc_strict_mode
2023-03-15T09:12:04.577365+02:00 use of GET_LOCK with pxc_strict_mode
2023-03-15T09:12:05.290288+02:00 use of RELEASE_LOCK with pxc_strict_mode
2023-03-15T09:12:05.296372+02:00 use of GET_LOCK with pxc_strict_mode
2023-03-15T09:12:05.310008+02:00 use of RELEASE_LOCK with pxc_strict_mode
2023-03-15T10:12:43.720576+02:00 Page cleaner took 5165ms to flush 2000
2023-03-15T10:12:51.514442+02:00 Page cleaner took 5793ms to flush 2000
2023-03-15T10:13:22.185343+02:00 Page cleaner took 4063ms to flush 2000
2023-03-15T13:13:34.584680+02:00 use of GET_LOCK with pxc_strict_mode
2023-03-15T13:13:34.597810+02:00 use of RELEASE_LOCK with pxc_strict_mode
2023-03-15T13:51:45.383215+02:00 use of GET_LOCK with pxc_strict_mode
2023-03-15T13:51:45.396164+02:00 use of RELEASE_LOCK with pxc_strict_mode
2023-03-15T15:44:04.523882+02:00 use of GET_LOCK with pxc_strict_mode
2023-03-15T15:44:04.534949+02:00 use of RELEASE_LOCK with pxc_strict_mode
2023-03-15T16:05:37.269541+02:00 use of GET_LOCK with pxc_strict_mode
2023-03-15T16:05:37.295035+02:00 use of RELEASE_LOCK with pxc_strict_mode
2023-03-15T20:10:57.784904+02:00 node2 joined
2023-03-15T20:10:57.785568+02:00 node3 left
2023-03-15T20:10:57.791959+02:00 node3 left
2023-03-15T20:10:57.797221+02:00 PRIMARY(n=2)
2023-03-15T20:18:57.060660+02:00 use of GET_LOCK with pxc_strict_mode
2023-03-15T20:18:57.081839+02:00 use of RELEASE_LOCK with pxc_strict_mode
2023-03-15T20:20:12.714291+02:00 node2 joined
2023-03-15T20:20:12.714331+02:00 node3 joined
2023-03-15T20:20:13.776977+02:00 PRIMARY(n=3)
2023-03-15T20:20:14.839684+02:00 local node will resync node3
2023-03-15T20:20:14.839723+02:00 SYNCED -> DONOR
2023-03-15T20:20:15.799020+02:00 IST will be used
2023-03-15T20:20:16.850525+02:00 finished sending IST to node3
2023-03-15T20:20:16.850549+02:00 DESYNCED -> JOINED
2023-03-15T20:20:16.865312+02:00 JOINED -> SYNCED
2023-03-15T20:39:36.865219+02:00 use of GET_LOCK with pxc_strict_mode
2023-03-15T20:39:36.946588+02:00 use of RELEASE_LOCK with pxc_strict_mode

1
src/go/pt-galera-log-explainer/tests/expected/operator_auto_ambiguous_ips_list_all_no_color
View File

@@ -505,6 +505,7 @@ mysql version 8.0.31 8.0.
2023-05-16T07:51:58.379294Z | recovering gcache |
2023-05-16T07:52:26.431252Z | CLOSED -> OPEN |
2023-05-16T07:52:26.431427Z | NON-PRIMARY(n=1) |
2023-05-16T07:52:26.931116Z | inactive check more than 1.5s (3.50171s) |
2023-05-16T10:03:18.655740Z starting(8.0.31) | |
2023-05-16T10:03:18.658268Z started(cluster) | |
2023-05-16T10:03:18.659239Z no grastate.dat file | |

1
src/go/pt-galera-log-explainer/types/regex.go
View File

@@ -71,6 +71,7 @@ var (
StatesRegexType RegexType = "states"
PXCOperatorRegexType RegexType = "pxc-operator"
ApplicativeRegexType RegexType = "applicative"
CustomRegexType RegexType = "custom"
)
type RegexMap map[string]*LogRegex