From 72fbe7496ca14194977bee716223c7b411c54324 Mon Sep 17 00:00:00 2001 From: Yoann La Cancellera Date: Wed, 18 Oct 2023 23:40:36 +0200 Subject: [PATCH] Add: operator member assocations regex It existed for non-operator setup, but was not working for operators due to k8s logs not interpreting newlines and tabs This operator version re-uses existing regular regex handlers directly --- .../pt-galera-log-explainer/regex/idents.go | 2 +- .../pt-galera-log-explainer/regex/operator.go | 34 ++++- .../regex/regex_test.go | 21 +++ ...operator_concurrent_ssts_list_all_no_color | 132 +++++++++--------- 4 files changed, 121 insertions(+), 68 deletions(-) diff --git a/src/go/pt-galera-log-explainer/regex/idents.go b/src/go/pt-galera-log-explainer/regex/idents.go index ae77fb58..46792404 100644 --- a/src/go/pt-galera-log-explainer/regex/idents.go +++ b/src/go/pt-galera-log-explainer/regex/idents.go @@ -42,7 +42,7 @@ var IdentsMap = types.RegexMap{ // 0: 015702fc-32f5-11ed-a4ca-267f97316394, node-1 // 1: 08dd5580-32f7-11ed-a9eb-af5e3d01519e, garb - // TO *never* DO: store indexes to later search for them using SST infos and STATES EXCHANGES logs. EDIT: is definitely NOT reliable + // TO *never* DO: store indexes to later search for them using SST infos and STATES EXCHANGES logs. They are definitely NOT reliable "RegexMemberAssociations": &types.LogRegex{ Regex: regexp.MustCompile("[0-9]: [a-z0-9]+-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]+, [a-zA-Z0-9-_]+"), InternalRegex: regexp.MustCompile(regexIdx + ": " + regexUUID + ", " + regexNodeName), diff --git a/src/go/pt-galera-log-explainer/regex/operator.go b/src/go/pt-galera-log-explainer/regex/operator.go index 480b0337..04214b40 100644 --- a/src/go/pt-galera-log-explainer/regex/operator.go +++ b/src/go/pt-galera-log-explainer/regex/operator.go @@ -47,9 +47,41 @@ var PXCOperatorMap = types.RegexMap{ "RegexGcacheScan": &types.LogRegex{ // those "operators" regexes do not have the log prefix added implicitely. It's not strictly needed, but // it will help to avoid catching random piece of log out of order - Regex: regexp.MustCompile("^{\"log\":\".*GCache::RingBuffer initial scan"), + Regex: regexp.MustCompile(k8sprefix + ".*GCache::RingBuffer initial scan"), Handler: func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) { return ctx, types.SimpleDisplayer("recovering gcache") }, }, + + // Unusual regex: because operators log does not handle newlines, it is contracted in a single line + // which the common "IdentsMap" regexes will miss. Even if they would catch it, it would only catch a single one, not all info + // so this regex is about capturing subgroups to re-handle each them to the appropriate existing IdentsMap regex + "RegexOperatorMemberAssociations": &types.LogRegex{ + Regex: regexp.MustCompile("================================================.*View:"), + InternalRegex: regexp.MustCompile("own_index: " + regexIdx + ".*(?P" + IdentsMap["RegexMemberCount"].Regex.String() + ")(?P(....-?[0-9]{1,2}(\\.-?[0-9])?: [a-z0-9]+-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]+, [a-zA-Z0-9-_\\.]+)+)"), + Handler: func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) { + + ctx.MyIdx = submatches[groupIdx] + + var ( + displayer types.LogDisplayer + msg string + ) + + ctx, displayer = IdentsMap["RegexMemberCount"].Handle(ctx, submatches["memberlog"]) + msg += displayer(ctx) + "; " + + subAssociations := strings.Split(submatches["compiledAssocations"], "\\n\\t") + if len(subAssociations) < 2 { + return ctx, types.SimpleDisplayer(msg) + } + for _, subAssocation := range subAssociations[1:] { + // better to reuse the idents regex + ctx, displayer = IdentsMap["RegexMemberAssociations"].Handle(ctx, subAssocation) + msg += displayer(ctx) + "; " + } + return ctx, types.SimpleDisplayer(msg) + }, + Verbosity: types.DebugMySQL, + }, } diff --git a/src/go/pt-galera-log-explainer/regex/regex_test.go b/src/go/pt-galera-log-explainer/regex/regex_test.go index 7cc1f033..e1c4e697 100644 --- a/src/go/pt-galera-log-explainer/regex/regex_test.go +++ b/src/go/pt-galera-log-explainer/regex/regex_test.go @@ -433,6 +433,27 @@ func TestRegexes(t *testing.T) { key: "RegexMemberAssociations", }, + { + log: "{\"log\":\"2001-01-01T01:01:01.000000Z 0 [Note] [MY-000000] [Galera] ================================================\\nView:\\n id: 9f191762-2542-11ee-89be-13bdb1218f0e:9375811\\n status: primary\\n protocol_version: 4\\n capabilities: MULTI-MASTER, CERTIFICATION, PARALLEL_APPLYING, REPLAY, ISOLATION, PAUSE, CAUSAL_READ, INCREMENTAL_WS, UNORDERED, PREORDERED, STREAMING, NBO\\n final: no\\n own_index: 0\\n members(3):\\n\\t0: 45406e8d-2de0-11ee-95fc-f29a5fdf1ee0, cluster1-0\\n\\t1: 5bf18376-2de0-11ee-8333-6e755a3456ca, cluster1-2\\n\\t2: 66e2b7bf-2de0-11ee-8000-f7d68b5cf6f6, cluster1-1\\n=================================================\\n\",\"file\":\"/var/lib/mysql/mysqld-error.log\"}", + inputCtx: types.LogCtx{ + OwnHashes: []string{}, + OwnNames: []string{}, + HashToNodeName: map[string]string{}, + }, + inputState: "PRIMARY", + expectedCtx: types.LogCtx{ + MyIdx: "0", + MemberCount: 3, + OwnHashes: []string{"45406e8d-95fc"}, + OwnNames: []string{"cluster1-0"}, + HashToNodeName: map[string]string{"45406e8d-95fc": "cluster1-0", "5bf18376-8333": "cluster1-2", "66e2b7bf-8000": "cluster1-1"}, + }, + expectedState: "PRIMARY", + expectedOut: "view member count: 3; 45406e8d-95fc is cluster1-0; 5bf18376-8333 is cluster1-2; 66e2b7bf-8000 is cluster1-1; ", + mapToTest: PXCOperatorMap, + key: "RegexOperatorMemberAssociations", + }, + { log: " members(1):", expectedOut: "view member count: 1", diff --git a/src/go/pt-galera-log-explainer/tests/expected/operator_concurrent_ssts_list_all_no_color b/src/go/pt-galera-log-explainer/tests/expected/operator_concurrent_ssts_list_all_no_color index 3197aae5..997f4ed2 100644 --- a/src/go/pt-galera-log-explainer/tests/expected/operator_concurrent_ssts_list_all_no_color +++ b/src/go/pt-galera-log-explainer/tests/expected/operator_concurrent_ssts_list_all_no_color @@ -10,9 +10,9 @@ mysql version 8.0.31 8. 2023-05-25T03:49:24.333464Z | | bootstrapping(empty grastate) 2023-05-25T03:49:24.333499Z | | safe_to_bootstrap: 1 2023-05-25T03:49:24.850000Z cluster1-2 joined | | -2023-05-25T03:49:24.850187Z 10.16.27.203 joined | | +2023-05-25T03:49:24.850187Z cluster1-1 joined | | 2023-05-25T03:49:24.850405Z | | cluster1-0 joined -2023-05-25T03:49:24.850417Z | | 10.16.27.203 joined +2023-05-25T03:49:24.850417Z | | cluster1-1 joined 2023-05-25T03:49:25.852441Z PRIMARY(n=3) | | 2023-05-25T03:49:26.348763Z | | CLOSED -> OPEN 2023-05-25T03:49:26.349013Z | | PRIMARY(n=3) @@ -29,7 +29,7 @@ mysql version 8.0.31 8. 2023-05-25T04:31:58.337408Z | | JOINER -> JOINED 2023-05-25T04:31:59.767489Z | | JOINED -> SYNCED 2023-05-25T04:34:20.398345Z | | received shutdown -2023-05-25T04:34:30.401445Z 10.16.27.203 joined | | +2023-05-25T04:34:30.401445Z cluster1-1 joined | | 2023-05-25T04:34:30.401532Z | | NON-PRIMARY(n=1) 2023-05-25T04:34:30.401594Z cluster1-2 left | | 2023-05-25T04:34:30.401727Z | | SYNCED -> OPEN @@ -49,9 +49,9 @@ mysql version 8.0.31 8. V 10.16.28.213 2023-05-25T04:35:01.848759Z | | cluster1-0 joined -2023-05-25T04:35:01.848785Z | | 10.16.27.203 joined +2023-05-25T04:35:01.848785Z | | cluster1-1 joined 2023-05-25T04:35:01.848877Z cluster1-2 joined | | -2023-05-25T04:35:01.848959Z 10.16.27.203 joined | | +2023-05-25T04:35:01.848959Z cluster1-1 joined | | 2023-05-25T04:35:01.850512Z PRIMARY(n=3) | | 2023-05-25T04:35:02.347755Z | | CLOSED -> OPEN 2023-05-25T04:35:02.347927Z | | PRIMARY(n=3) @@ -68,11 +68,11 @@ mysql version 8.0.31 8. 2023-05-25T04:35:15.078178Z | | JOINED -> SYNCED 2023-05-25T04:35:30.559038Z | | cluster1-0 joined 2023-05-25T04:35:30.559192Z cluster1-2 joined | | -2023-05-25T04:35:30.559318Z 10.16.27.203 left | | -2023-05-25T04:35:30.559684Z | | 10.16.27.203 left -2023-05-25T04:35:30.560761Z 10.16.27.203 left | | +2023-05-25T04:35:30.559318Z cluster1-1 left | | +2023-05-25T04:35:30.559684Z | | cluster1-1 left +2023-05-25T04:35:30.560761Z cluster1-1 left | | 2023-05-25T04:35:30.560850Z PRIMARY(n=2) | | -2023-05-25T04:35:30.561080Z | | 10.16.27.203 left +2023-05-25T04:35:30.561080Z | | cluster1-1 left 2023-05-25T04:35:30.561149Z | | PRIMARY(n=2) 2023-05-25T04:36:12.725113Z | starting(8.0.31) | 2023-05-25T04:36:12.735830Z | started(cluster) | @@ -153,13 +153,13 @@ mysql version 8.0.31 8. 2023-05-25T04:38:14.953589Z JOINER -> JOINED | | 2023-05-25T04:38:14.954030Z JOINED -> SYNCED | | 2023-05-26T03:00:22.581588Z | cluster1-0 joined | -2023-05-26T03:00:22.581645Z | 10.16.29.103 joined | +2023-05-26T03:00:22.581645Z | garb joined | 2023-05-26T03:00:22.581658Z | cluster1-2 joined | -2023-05-26T03:00:22.581847Z 10.16.29.103 joined | | +2023-05-26T03:00:22.581847Z garb joined | | 2023-05-26T03:00:22.581912Z | | cluster1-0 joined 2023-05-26T03:00:22.581946Z cluster1-2 joined | | 2023-05-26T03:00:22.581970Z cluster1-1 joined | | -2023-05-26T03:00:22.582016Z | | 10.16.29.103 joined +2023-05-26T03:00:22.582016Z | | garb joined 2023-05-26T03:00:22.582040Z | | cluster1-1 joined 2023-05-26T03:00:22.583782Z | PRIMARY(n=4) | 2023-05-26T03:00:22.583952Z | | PRIMARY(n=4) @@ -168,7 +168,7 @@ mysql version 8.0.31 8. 2023-05-26T03:00:23.082696Z | | local node will resync garb 2023-05-26T03:00:23.082706Z | | SYNCED -> DONOR 2023-05-26T03:00:23.082747Z cluster1-2 will resync garb | | -2023-05-26T03:00:34.572248Z | | SST to 10.16.29.103 +2023-05-26T03:00:34.572248Z | | SST to garb 2023-05-26T03:40:23.808519Z | | finished sending SST to garb 2023-05-26T03:40:23.808574Z | | DESYNCED -> JOINED 2023-05-26T03:40:23.808600Z cluster1-2 synced garb | | @@ -178,34 +178,34 @@ mysql version 8.0.31 8. 2023-05-26T03:40:26.047565Z cluster1-2 joined | | 2023-05-26T03:40:26.047613Z | | cluster1-1 joined 2023-05-26T03:40:26.047651Z cluster1-1 joined | | -2023-05-26T03:40:26.047664Z | | 10.16.29.103 left -2023-05-26T03:40:26.047712Z 10.16.29.103 left | | +2023-05-26T03:40:26.047664Z | | garb left +2023-05-26T03:40:26.047712Z garb left | | 2023-05-26T03:40:26.047973Z | cluster1-0 joined | 2023-05-26T03:40:26.048058Z | cluster1-2 joined | -2023-05-26T03:40:26.048117Z | 10.16.29.103 left | -2023-05-26T03:40:26.049387Z 10.16.29.103 left | | -2023-05-26T03:40:26.049412Z | | 10.16.29.103 left +2023-05-26T03:40:26.048117Z | garb left | +2023-05-26T03:40:26.049387Z garb left | | +2023-05-26T03:40:26.049412Z | | garb left 2023-05-26T03:40:26.049454Z PRIMARY(n=3) | | 2023-05-26T03:40:26.049477Z | | PRIMARY(n=3) -2023-05-26T03:40:26.049765Z | 10.16.29.103 left | +2023-05-26T03:40:26.049765Z | garb left | 2023-05-26T03:40:26.049947Z | PRIMARY(n=3) | 2023-05-27T03:00:19.537730Z | cluster1-0 joined | 2023-05-27T03:00:19.537767Z | cluster1-2 joined | -2023-05-27T03:00:19.537775Z | 10.16.29.248 joined | +2023-05-27T03:00:19.537775Z | garb joined | 2023-05-27T03:00:19.537993Z cluster1-2 joined | | -2023-05-27T03:00:19.538086Z 10.16.29.248 joined | | +2023-05-27T03:00:19.538086Z garb joined | | 2023-05-27T03:00:19.538110Z cluster1-1 joined | | 2023-05-27T03:00:19.539621Z | PRIMARY(n=4) | 2023-05-27T03:00:19.539965Z PRIMARY(n=4) | | 2023-05-27T03:00:19.542417Z | | cluster1-0 joined -2023-05-27T03:00:19.542503Z | | 10.16.29.248 joined +2023-05-27T03:00:19.542503Z | | garb joined 2023-05-27T03:00:19.542526Z | | cluster1-1 joined 2023-05-27T03:00:19.544132Z | | PRIMARY(n=4) 2023-05-27T03:00:20.038432Z | cluster1-2 will resync garb | 2023-05-27T03:00:20.038598Z cluster1-2 will resync garb | | 2023-05-27T03:00:20.043068Z | | local node will resync garb 2023-05-27T03:00:20.043091Z | | SYNCED -> DONOR -2023-05-27T03:00:31.529154Z | | SST to 10.16.29.248 +2023-05-27T03:00:31.529154Z | | SST to garb 2023-05-27T03:39:48.765020Z cluster1-2 synced garb | | 2023-05-27T03:39:48.765166Z | cluster1-2 synced garb | 2023-05-27T03:39:48.768602Z | | finished sending SST to garb @@ -213,28 +213,28 @@ mysql version 8.0.31 8. 2023-05-27T03:39:48.769112Z | | JOINED -> SYNCED 2023-05-27T03:39:51.224587Z cluster1-2 joined | | 2023-05-27T03:39:51.224622Z cluster1-1 joined | | -2023-05-27T03:39:51.224631Z 10.16.29.248 left | | +2023-05-27T03:39:51.224631Z garb left | | 2023-05-27T03:39:51.224821Z | cluster1-0 joined | 2023-05-27T03:39:51.224991Z | cluster1-2 joined | -2023-05-27T03:39:51.225009Z | 10.16.29.248 left | -2023-05-27T03:39:51.226003Z 10.16.29.248 left | | -2023-05-27T03:39:51.226069Z | 10.16.29.248 left | +2023-05-27T03:39:51.225009Z | garb left | +2023-05-27T03:39:51.226003Z garb left | | +2023-05-27T03:39:51.226069Z | garb left | 2023-05-27T03:39:51.226111Z | PRIMARY(n=3) | 2023-05-27T03:39:51.226160Z PRIMARY(n=3) | | 2023-05-27T03:39:51.228020Z | | cluster1-0 joined 2023-05-27T03:39:51.228100Z | | cluster1-1 joined -2023-05-27T03:39:51.228115Z | | 10.16.29.248 left -2023-05-27T03:39:51.229544Z | | 10.16.29.248 left +2023-05-27T03:39:51.228115Z | | garb left +2023-05-27T03:39:51.229544Z | | garb left 2023-05-27T03:39:51.229698Z | | PRIMARY(n=3) 2023-05-28T03:00:20.179665Z cluster1-2 joined | | 2023-05-28T03:00:20.179723Z cluster1-1 joined | | -2023-05-28T03:00:20.179744Z 10.16.29.190 joined | | +2023-05-28T03:00:20.179744Z garb joined | | 2023-05-28T03:00:20.180516Z | cluster1-0 joined | 2023-05-28T03:00:20.180583Z | cluster1-2 joined | -2023-05-28T03:00:20.180597Z | 10.16.29.190 joined | +2023-05-28T03:00:20.180597Z | garb joined | 2023-05-28T03:00:20.180757Z | | cluster1-0 joined 2023-05-28T03:00:20.180839Z | | cluster1-1 joined -2023-05-28T03:00:20.180863Z | | 10.16.29.190 joined +2023-05-28T03:00:20.180863Z | | garb joined 2023-05-28T03:00:20.181677Z PRIMARY(n=4) | | 2023-05-28T03:00:20.182305Z | PRIMARY(n=4) | 2023-05-28T03:00:20.182540Z | | PRIMARY(n=4) @@ -242,7 +242,7 @@ mysql version 8.0.31 8. 2023-05-28T03:00:20.681257Z | cluster1-2 will resync garb | 2023-05-28T03:00:20.681447Z | | local node will resync garb 2023-05-28T03:00:20.681471Z | | SYNCED -> DONOR -2023-05-28T03:00:32.184853Z | | SST to 10.16.29.190 +2023-05-28T03:00:32.184853Z | | SST to garb 2023-05-28T03:40:51.321660Z | | finished sending SST to garb 2023-05-28T03:40:51.321733Z | | DESYNCED -> JOINED 2023-05-28T03:40:51.321783Z cluster1-2 synced garb | | @@ -250,18 +250,18 @@ mysql version 8.0.31 8. 2023-05-28T03:40:51.322593Z | | JOINED -> SYNCED 2023-05-28T03:40:53.656940Z | | cluster1-0 joined 2023-05-28T03:40:53.656966Z | | cluster1-1 joined -2023-05-28T03:40:53.656981Z | | 10.16.29.190 left +2023-05-28T03:40:53.656981Z | | garb left 2023-05-28T03:40:53.657404Z | cluster1-0 joined | 2023-05-28T03:40:53.657428Z | cluster1-2 joined | -2023-05-28T03:40:53.657443Z | 10.16.29.190 left | +2023-05-28T03:40:53.657443Z | garb left | 2023-05-28T03:40:53.657819Z cluster1-2 joined | | 2023-05-28T03:40:53.657840Z cluster1-1 joined | | -2023-05-28T03:40:53.657847Z 10.16.29.190 left | | -2023-05-28T03:40:53.658835Z 10.16.29.190 left | | -2023-05-28T03:40:53.658995Z | | 10.16.29.190 left +2023-05-28T03:40:53.657847Z garb left | | +2023-05-28T03:40:53.658835Z garb left | | +2023-05-28T03:40:53.658995Z | | garb left 2023-05-28T03:40:53.659027Z PRIMARY(n=3) | | 2023-05-28T03:40:53.659156Z | | PRIMARY(n=3) -2023-05-28T03:40:53.659348Z | 10.16.29.190 left | +2023-05-28T03:40:53.659348Z | garb left | 2023-05-28T03:40:53.659384Z | PRIMARY(n=3) | 2023-05-28T08:09:01.368781Z | received shutdown | 2023-05-28T08:19:06.122140Z cluster1-1 suspected to be down | | @@ -459,15 +459,15 @@ mysql version 8.0.31 8. 2023-05-29T05:00:37.738615Z | | garb joined 2023-05-29T05:00:37.738711Z | | cluster1-0 joined 2023-05-29T05:00:37.738735Z | | cluster1-1 joined -2023-05-29T05:00:37.738755Z | | 10.16.29.220 joined +2023-05-29T05:00:37.738755Z | | garb joined 2023-05-29T05:00:37.740237Z garb joined | | 2023-05-29T05:00:37.740269Z | garb joined | 2023-05-29T05:00:37.740320Z cluster1-1 joined | | 2023-05-29T05:00:37.740345Z cluster1-2 joined | | -2023-05-29T05:00:37.740366Z 10.16.29.220 joined | | +2023-05-29T05:00:37.740366Z garb joined | | 2023-05-29T05:00:37.740402Z | cluster1-0 joined | 2023-05-29T05:00:37.740430Z | cluster1-2 joined | -2023-05-29T05:00:37.740475Z | 10.16.29.220 joined | +2023-05-29T05:00:37.740475Z | garb joined | 2023-05-29T05:00:37.740651Z | | PRIMARY(n=5) 2023-05-29T05:00:37.742138Z | PRIMARY(n=5) | 2023-05-29T05:00:37.742177Z PRIMARY(n=5) | | @@ -475,7 +475,7 @@ mysql version 8.0.31 8. 2023-05-29T05:00:38.239874Z | | SYNCED -> DONOR 2023-05-29T05:00:38.241430Z | cluster1-2 will resync garb | 2023-05-29T05:00:38.241476Z cluster1-2 will resync garb | | -2023-05-29T05:00:49.731756Z | | SST to 10.16.29.220 +2023-05-29T05:00:49.731756Z | | SST to garb 2023-05-29T05:54:56.049218Z | | finished sending SST to garb 2023-05-29T05:54:56.049291Z | | DESYNCED -> JOINED 2023-05-29T05:54:56.049762Z | | JOINED -> SYNCED @@ -484,15 +484,15 @@ mysql version 8.0.31 8. 2023-05-29T05:54:58.219414Z | | garb joined 2023-05-29T05:54:58.219474Z | | cluster1-0 joined 2023-05-29T05:54:58.219489Z | | cluster1-1 joined -2023-05-29T05:54:58.219504Z | | 10.16.29.220 left +2023-05-29T05:54:58.219504Z | | garb left 2023-05-29T05:54:58.220860Z garb joined | | 2023-05-29T05:54:58.220913Z cluster1-1 joined | | 2023-05-29T05:54:58.220925Z cluster1-2 joined | | -2023-05-29T05:54:58.220954Z 10.16.29.220 left | | +2023-05-29T05:54:58.220954Z garb left | | 2023-05-29T05:54:58.221144Z | garb joined | 2023-05-29T05:54:58.221241Z | cluster1-0 joined | 2023-05-29T05:54:58.221263Z | cluster1-2 joined | -2023-05-29T05:54:58.221280Z | 10.16.29.220 left | +2023-05-29T05:54:58.221280Z | garb left | 2023-05-29T05:55:03.220834Z | | garb suspected to be down 2023-05-29T05:55:03.222309Z garb suspected to be down | | 2023-05-29T05:55:03.722761Z | garb suspected to be down | @@ -510,13 +510,13 @@ mysql version 8.0.31 8. 2023-05-29T05:55:04.225876Z PRIMARY(n=3) | | 2023-05-29T06:18:41.339691Z | | cluster1-0 joined 2023-05-29T06:18:41.339777Z | | cluster1-1 joined -2023-05-29T06:18:41.339802Z | | 10.16.27.183 joined +2023-05-29T06:18:41.339802Z | | garb joined 2023-05-29T06:18:41.340638Z cluster1-1 joined | | 2023-05-29T06:18:41.340748Z cluster1-2 joined | | -2023-05-29T06:18:41.340777Z 10.16.27.183 joined | | +2023-05-29T06:18:41.340777Z garb joined | | 2023-05-29T06:18:41.340846Z | cluster1-0 joined | 2023-05-29T06:18:41.340903Z | cluster1-2 joined | -2023-05-29T06:18:41.340914Z | 10.16.27.183 joined | +2023-05-29T06:18:41.340914Z | garb joined | 2023-05-29T06:18:41.341398Z | | PRIMARY(n=4) 2023-05-29T06:18:41.342417Z | PRIMARY(n=4) | 2023-05-29T06:18:41.342618Z PRIMARY(n=4) | | @@ -524,19 +524,19 @@ mysql version 8.0.31 8. 2023-05-29T06:18:41.840393Z | | SYNCED -> DONOR 2023-05-29T06:18:41.841468Z | cluster1-2 will resync garb | 2023-05-29T06:18:41.841677Z cluster1-2 will resync garb | | -2023-05-29T06:18:53.370532Z | | SST to 10.16.27.183 +2023-05-29T06:18:53.370532Z | | SST to garb 2023-05-29T06:21:16.932469Z | | garb joined 2023-05-29T06:21:16.932542Z | | cluster1-0 joined 2023-05-29T06:21:16.932559Z | | cluster1-1 joined -2023-05-29T06:21:16.932574Z | | 10.16.27.183 joined +2023-05-29T06:21:16.932574Z | | garb joined 2023-05-29T06:21:16.933460Z | garb joined | 2023-05-29T06:21:16.933530Z | cluster1-0 joined | 2023-05-29T06:21:16.933543Z | cluster1-2 joined | -2023-05-29T06:21:16.933554Z | 10.16.27.183 joined | +2023-05-29T06:21:16.933554Z | garb joined | 2023-05-29T06:21:16.933921Z garb joined | | 2023-05-29T06:21:16.933996Z cluster1-1 joined | | 2023-05-29T06:21:16.934011Z cluster1-2 joined | | -2023-05-29T06:21:16.934022Z 10.16.27.183 joined | | +2023-05-29T06:21:16.934022Z garb joined | | 2023-05-29T06:21:16.934118Z | | PRIMARY(n=5) 2023-05-29T06:21:16.935175Z | PRIMARY(n=5) | 2023-05-29T06:21:16.935415Z PRIMARY(n=5) | | @@ -553,44 +553,44 @@ mysql version 8.0.31 8. 2023-05-29T07:16:33.678136Z | | garb joined 2023-05-29T07:16:33.678217Z | | cluster1-0 joined 2023-05-29T07:16:33.678236Z | | cluster1-1 joined -2023-05-29T07:16:33.678252Z | | 10.16.27.183 left +2023-05-29T07:16:33.678252Z | | garb left 2023-05-29T07:16:34.694046Z | SST error | 2023-05-29T07:16:38.678115Z garb suspected to be down | | 2023-05-29T07:16:38.678140Z cluster1-2 suspected to be down | | 2023-05-29T07:16:38.678174Z | garb suspected to be down | -2023-05-29T07:16:38.678187Z 10.16.27.183 suspected to be down | | +2023-05-29T07:16:38.678187Z garb suspected to be down | | 2023-05-29T07:16:38.678199Z | cluster1-2 suspected to be down | -2023-05-29T07:16:38.678232Z | 10.16.27.183 suspected to be down | +2023-05-29T07:16:38.678232Z | garb suspected to be down | 2023-05-29T07:16:38.679876Z | | garb suspected to be down 2023-05-29T07:16:38.679923Z | | cluster1-0 suspected to be down 2023-05-29T07:16:38.679965Z | | cluster1-1 suspected to be down 2023-05-29T07:16:39.178309Z garb suspected to be down | | 2023-05-29T07:16:39.178383Z | garb suspected to be down | 2023-05-29T07:16:39.178407Z cluster1-2 suspected to be down | | -2023-05-29T07:16:39.178437Z 10.16.27.183 suspected to be down | | +2023-05-29T07:16:39.178437Z garb suspected to be down | | 2023-05-29T07:16:39.178468Z | cluster1-2 suspected to be down | -2023-05-29T07:16:39.178496Z | 10.16.27.183 suspected to be down | +2023-05-29T07:16:39.178496Z | garb suspected to be down | 2023-05-29T07:16:39.678585Z | garb suspected to be down | 2023-05-29T07:16:39.678620Z garb suspected to be down | | 2023-05-29T07:16:39.678649Z | cluster1-2 suspected to be down | -2023-05-29T07:16:39.678672Z | 10.16.27.183 suspected to be down | +2023-05-29T07:16:39.678672Z | garb suspected to be down | 2023-05-29T07:16:39.678717Z cluster1-2 suspected to be down | | -2023-05-29T07:16:39.678763Z 10.16.27.183 suspected to be down | | +2023-05-29T07:16:39.678763Z garb suspected to be down | | 2023-05-29T07:16:39.680986Z | | NON-PRIMARY(n=1) 2023-05-29T07:16:39.681204Z | | SYNCED -> OPEN 2023-05-29T07:16:39.681241Z | | NON-PRIMARY(n=1) 2023-05-29T07:16:40.178884Z garb suspected to be down | | 2023-05-29T07:16:40.178983Z cluster1-2 suspected to be down | | -2023-05-29T07:16:40.179015Z 10.16.27.183 suspected to be down | | +2023-05-29T07:16:40.179015Z garb suspected to be down | | 2023-05-29T07:16:40.179037Z | garb suspected to be down | 2023-05-29T07:16:40.179126Z | cluster1-2 suspected to be down | -2023-05-29T07:16:40.179154Z | 10.16.27.183 suspected to be down | +2023-05-29T07:16:40.179154Z | garb suspected to be down | 2023-05-29T07:16:40.679187Z garb suspected to be down | | 2023-05-29T07:16:40.679234Z | garb suspected to be down | 2023-05-29T07:16:40.679286Z | cluster1-2 suspected to be down | 2023-05-29T07:16:40.679287Z cluster1-2 suspected to be down | | -2023-05-29T07:16:40.679299Z | 10.16.27.183 suspected to be down | -2023-05-29T07:16:40.679313Z 10.16.27.183 suspected to be down | | +2023-05-29T07:16:40.679299Z | garb suspected to be down | +2023-05-29T07:16:40.679313Z garb suspected to be down | | 2023-05-29T07:16:41.679431Z garb suspected to be down | | 2023-05-29T07:16:41.679447Z cluster1-2 suspected to be down | | 2023-05-29T07:16:41.679615Z | garb suspected to be down | @@ -647,11 +647,11 @@ mysql version 8.0.31 8. 2023-05-29T07:16:48.182140Z cluster1-2 suspected to be down | | 2023-05-29T07:16:48.183482Z | garb suspected to be down | 2023-05-29T07:16:48.183563Z | cluster1-2 suspected to be down | -2023-05-29T07:16:48.682756Z | 10.16.27.183 left | +2023-05-29T07:16:48.682756Z | garb left | 2023-05-29T07:16:48.682898Z | NON-PRIMARY(n=1) | 2023-05-29T07:16:48.682999Z | DESYNCED -> OPEN | 2023-05-29T07:16:48.683014Z | NON-PRIMARY(n=1) | -2023-05-29T07:16:48.683734Z 10.16.27.183 left | | +2023-05-29T07:16:48.683734Z garb left | | 2023-05-29T07:16:48.683956Z NON-PRIMARY(n=1) | | 2023-05-29T07:16:48.684193Z SYNCED -> OPEN | | 2023-05-29T07:16:48.684230Z NON-PRIMARY(n=1) | |