percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-23 12:47:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add: operator member assocations regex

Browse Source 
It existed for non-operator setup, but was not working for operators due
to k8s logs not interpreting newlines and tabs
This operator version re-uses existing regular regex handlers directly
This commit is contained in:
Yoann La Cancellera
2023-10-18 23:40:36 +02:00
committed by Sveta Smirnova
parent 4c8fe1d832
commit 72fbe7496c
4 changed files with 121 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/go/pt-galera-log-explainer/regex/idents.go
View File

@@ -42,7 +42,7 @@ var IdentsMap = types.RegexMap{
// 0: 015702fc-32f5-11ed-a4ca-267f97316394, node-1 // 0: 015702fc-32f5-11ed-a4ca-267f97316394, node-1
// 1: 08dd5580-32f7-11ed-a9eb-af5e3d01519e, garb // 1: 08dd5580-32f7-11ed-a9eb-af5e3d01519e, garb
// TO *never* DO: store indexes to later search for them using SST infos and STATES EXCHANGES logs. EDIT: is definitely NOT reliable // TO *never* DO: store indexes to later search for them using SST infos and STATES EXCHANGES logs. They are definitely NOT reliable
"RegexMemberAssociations": &types.LogRegex{ "RegexMemberAssociations": &types.LogRegex{
Regex: regexp.MustCompile("[0-9]: [a-z0-9]+-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]+, [a-zA-Z0-9-_]+"), Regex: regexp.MustCompile("[0-9]: [a-z0-9]+-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]+, [a-zA-Z0-9-_]+"),
InternalRegex: regexp.MustCompile(regexIdx + ": " + regexUUID + ", " + regexNodeName), InternalRegex: regexp.MustCompile(regexIdx + ": " + regexUUID + ", " + regexNodeName),

34
src/go/pt-galera-log-explainer/regex/operator.go
View File

@@ -47,9 +47,41 @@ var PXCOperatorMap = types.RegexMap{
"RegexGcacheScan": &types.LogRegex{ "RegexGcacheScan": &types.LogRegex{
// those "operators" regexes do not have the log prefix added implicitely. It's not strictly needed, but // those "operators" regexes do not have the log prefix added implicitely. It's not strictly needed, but
// it will help to avoid catching random piece of log out of order // it will help to avoid catching random piece of log out of order
Regex: regexp.MustCompile("^{\"log\":\".*GCache::RingBuffer initial scan"), Regex: regexp.MustCompile(k8sprefix + ".*GCache::RingBuffer initial scan"),
Handler: func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) { Handler: func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) {
return ctx, types.SimpleDisplayer("recovering gcache") return ctx, types.SimpleDisplayer("recovering gcache")
}, },
}, },
// Unusual regex: because operators log does not handle newlines, it is contracted in a single line
// which the common "IdentsMap" regexes will miss. Even if they would catch it, it would only catch a single one, not all info
// so this regex is about capturing subgroups to re-handle each them to the appropriate existing IdentsMap regex
"RegexOperatorMemberAssociations": &types.LogRegex{
Regex: regexp.MustCompile("================================================.*View:"),
InternalRegex: regexp.MustCompile("own_index: " + regexIdx + ".*(?P<memberlog>" + IdentsMap["RegexMemberCount"].Regex.String() + ")(?P<compiledAssocations>(....-?[0-9]{1,2}(\\.-?[0-9])?: [a-z0-9]+-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]+, [a-zA-Z0-9-_\\.]+)+)"),
Handler: func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) {
ctx.MyIdx = submatches[groupIdx]
var (
displayer types.LogDisplayer
msg string
)
ctx, displayer = IdentsMap["RegexMemberCount"].Handle(ctx, submatches["memberlog"])
msg += displayer(ctx) + "; "
subAssociations := strings.Split(submatches["compiledAssocations"], "\\n\\t")
if len(subAssociations) < 2 {
return ctx, types.SimpleDisplayer(msg)
}
for _, subAssocation := range subAssociations[1:] {
// better to reuse the idents regex
ctx, displayer = IdentsMap["RegexMemberAssociations"].Handle(ctx, subAssocation)
msg += displayer(ctx) + "; "
}
return ctx, types.SimpleDisplayer(msg)
},
Verbosity: types.DebugMySQL,
},
} }

21
src/go/pt-galera-log-explainer/regex/regex_test.go
View File

@@ -433,6 +433,27 @@ func TestRegexes(t *testing.T) {
key: "RegexMemberAssociations", key: "RegexMemberAssociations",
}, },
{
log: "{\"log\":\"2001-01-01T01:01:01.000000Z 0 [Note] [MY-000000] [Galera] ================================================\\nView:\\n id: 9f191762-2542-11ee-89be-13bdb1218f0e:9375811\\n status: primary\\n protocol_version: 4\\n capabilities: MULTI-MASTER, CERTIFICATION, PARALLEL_APPLYING, REPLAY, ISOLATION, PAUSE, CAUSAL_READ, INCREMENTAL_WS, UNORDERED, PREORDERED, STREAMING, NBO\\n final: no\\n own_index: 0\\n members(3):\\n\\t0: 45406e8d-2de0-11ee-95fc-f29a5fdf1ee0, cluster1-0\\n\\t1: 5bf18376-2de0-11ee-8333-6e755a3456ca, cluster1-2\\n\\t2: 66e2b7bf-2de0-11ee-8000-f7d68b5cf6f6, cluster1-1\\n=================================================\\n\",\"file\":\"/var/lib/mysql/mysqld-error.log\"}",
inputCtx: types.LogCtx{
OwnHashes: []string{},
OwnNames: []string{},
HashToNodeName: map[string]string{},
},
inputState: "PRIMARY",
expectedCtx: types.LogCtx{
MyIdx: "0",
MemberCount: 3,
OwnHashes: []string{"45406e8d-95fc"},
OwnNames: []string{"cluster1-0"},
HashToNodeName: map[string]string{"45406e8d-95fc": "cluster1-0", "5bf18376-8333": "cluster1-2", "66e2b7bf-8000": "cluster1-1"},
},
expectedState: "PRIMARY",
expectedOut: "view member count: 3; 45406e8d-95fc is cluster1-0; 5bf18376-8333 is cluster1-2; 66e2b7bf-8000 is cluster1-1; ",
mapToTest: PXCOperatorMap,
key: "RegexOperatorMemberAssociations",
},
{ {
log: " members(1):", log: " members(1):",
expectedOut: "view member count: 1", expectedOut: "view member count: 1",

132
src/go/pt-galera-log-explainer/tests/expected/operator_concurrent_ssts_list_all_no_color
View File

@@ -10,9 +10,9 @@ mysql version 8.0.31 8.
2023-05-25T03:49:24.333464Z | | bootstrapping(empty grastate) 2023-05-25T03:49:24.333464Z | | bootstrapping(empty grastate)
2023-05-25T03:49:24.333499Z | | safe_to_bootstrap: 1 2023-05-25T03:49:24.333499Z | | safe_to_bootstrap: 1
2023-05-25T03:49:24.850000Z cluster1-2 joined | | 2023-05-25T03:49:24.850000Z cluster1-2 joined | |
2023-05-25T03:49:24.850187Z 10.16.27.203 joined | | 2023-05-25T03:49:24.850187Z cluster1-1 joined | |
2023-05-25T03:49:24.850405Z | | cluster1-0 joined 2023-05-25T03:49:24.850405Z | | cluster1-0 joined
2023-05-25T03:49:24.850417Z | | 10.16.27.203 joined 2023-05-25T03:49:24.850417Z | | cluster1-1 joined
2023-05-25T03:49:25.852441Z PRIMARY(n=3) | | 2023-05-25T03:49:25.852441Z PRIMARY(n=3) | |
2023-05-25T03:49:26.348763Z | | CLOSED -> OPEN 2023-05-25T03:49:26.348763Z | | CLOSED -> OPEN
2023-05-25T03:49:26.349013Z | | PRIMARY(n=3) 2023-05-25T03:49:26.349013Z | | PRIMARY(n=3)
@@ -29,7 +29,7 @@ mysql version 8.0.31 8.
2023-05-25T04:31:58.337408Z | | JOINER -> JOINED 2023-05-25T04:31:58.337408Z | | JOINER -> JOINED
2023-05-25T04:31:59.767489Z | | JOINED -> SYNCED 2023-05-25T04:31:59.767489Z | | JOINED -> SYNCED
2023-05-25T04:34:20.398345Z | | received shutdown 2023-05-25T04:34:20.398345Z | | received shutdown
2023-05-25T04:34:30.401445Z 10.16.27.203 joined | | 2023-05-25T04:34:30.401445Z cluster1-1 joined | |
2023-05-25T04:34:30.401532Z | | NON-PRIMARY(n=1) 2023-05-25T04:34:30.401532Z | | NON-PRIMARY(n=1)
2023-05-25T04:34:30.401594Z cluster1-2 left | | 2023-05-25T04:34:30.401594Z cluster1-2 left | |
2023-05-25T04:34:30.401727Z | | SYNCED -> OPEN 2023-05-25T04:34:30.401727Z | | SYNCED -> OPEN
@@ -49,9 +49,9 @@ mysql version 8.0.31 8.
V V
10.16.28.213 10.16.28.213
2023-05-25T04:35:01.848759Z | | cluster1-0 joined 2023-05-25T04:35:01.848759Z | | cluster1-0 joined
2023-05-25T04:35:01.848785Z | | 10.16.27.203 joined 2023-05-25T04:35:01.848785Z | | cluster1-1 joined
2023-05-25T04:35:01.848877Z cluster1-2 joined | | 2023-05-25T04:35:01.848877Z cluster1-2 joined | |
2023-05-25T04:35:01.848959Z 10.16.27.203 joined | | 2023-05-25T04:35:01.848959Z cluster1-1 joined | |
2023-05-25T04:35:01.850512Z PRIMARY(n=3) | | 2023-05-25T04:35:01.850512Z PRIMARY(n=3) | |
2023-05-25T04:35:02.347755Z | | CLOSED -> OPEN 2023-05-25T04:35:02.347755Z | | CLOSED -> OPEN
2023-05-25T04:35:02.347927Z | | PRIMARY(n=3) 2023-05-25T04:35:02.347927Z | | PRIMARY(n=3)
@@ -68,11 +68,11 @@ mysql version 8.0.31 8.
2023-05-25T04:35:15.078178Z | | JOINED -> SYNCED 2023-05-25T04:35:15.078178Z | | JOINED -> SYNCED
2023-05-25T04:35:30.559038Z | | cluster1-0 joined 2023-05-25T04:35:30.559038Z | | cluster1-0 joined
2023-05-25T04:35:30.559192Z cluster1-2 joined | | 2023-05-25T04:35:30.559192Z cluster1-2 joined | |
2023-05-25T04:35:30.559318Z 10.16.27.203 left | | 2023-05-25T04:35:30.559318Z cluster1-1 left | |
2023-05-25T04:35:30.559684Z | | 10.16.27.203 left 2023-05-25T04:35:30.559684Z | | cluster1-1 left
2023-05-25T04:35:30.560761Z 10.16.27.203 left | | 2023-05-25T04:35:30.560761Z cluster1-1 left | |
2023-05-25T04:35:30.560850Z PRIMARY(n=2) | | 2023-05-25T04:35:30.560850Z PRIMARY(n=2) | |
2023-05-25T04:35:30.561080Z | | 10.16.27.203 left 2023-05-25T04:35:30.561080Z | | cluster1-1 left
2023-05-25T04:35:30.561149Z | | PRIMARY(n=2) 2023-05-25T04:35:30.561149Z | | PRIMARY(n=2)
2023-05-25T04:36:12.725113Z | starting(8.0.31) | 2023-05-25T04:36:12.725113Z | starting(8.0.31) |
2023-05-25T04:36:12.735830Z | started(cluster) | 2023-05-25T04:36:12.735830Z | started(cluster) |
@@ -153,13 +153,13 @@ mysql version 8.0.31 8.
2023-05-25T04:38:14.953589Z JOINER -> JOINED | | 2023-05-25T04:38:14.953589Z JOINER -> JOINED | |
2023-05-25T04:38:14.954030Z JOINED -> SYNCED | | 2023-05-25T04:38:14.954030Z JOINED -> SYNCED | |
2023-05-26T03:00:22.581588Z | cluster1-0 joined | 2023-05-26T03:00:22.581588Z | cluster1-0 joined |
2023-05-26T03:00:22.581645Z | 10.16.29.103 joined | 2023-05-26T03:00:22.581645Z | garb joined |
2023-05-26T03:00:22.581658Z | cluster1-2 joined | 2023-05-26T03:00:22.581658Z | cluster1-2 joined |
2023-05-26T03:00:22.581847Z 10.16.29.103 joined | | 2023-05-26T03:00:22.581847Z garb joined | |
2023-05-26T03:00:22.581912Z | | cluster1-0 joined 2023-05-26T03:00:22.581912Z | | cluster1-0 joined
2023-05-26T03:00:22.581946Z cluster1-2 joined | | 2023-05-26T03:00:22.581946Z cluster1-2 joined | |
2023-05-26T03:00:22.581970Z cluster1-1 joined | | 2023-05-26T03:00:22.581970Z cluster1-1 joined | |
2023-05-26T03:00:22.582016Z | | 10.16.29.103 joined 2023-05-26T03:00:22.582016Z | | garb joined
2023-05-26T03:00:22.582040Z | | cluster1-1 joined 2023-05-26T03:00:22.582040Z | | cluster1-1 joined
2023-05-26T03:00:22.583782Z | PRIMARY(n=4) | 2023-05-26T03:00:22.583782Z | PRIMARY(n=4) |
2023-05-26T03:00:22.583952Z | | PRIMARY(n=4) 2023-05-26T03:00:22.583952Z | | PRIMARY(n=4)
@@ -168,7 +168,7 @@ mysql version 8.0.31 8.
2023-05-26T03:00:23.082696Z | | local node will resync garb 2023-05-26T03:00:23.082696Z | | local node will resync garb
2023-05-26T03:00:23.082706Z | | SYNCED -> DONOR 2023-05-26T03:00:23.082706Z | | SYNCED -> DONOR
2023-05-26T03:00:23.082747Z cluster1-2 will resync garb | | 2023-05-26T03:00:23.082747Z cluster1-2 will resync garb | |
2023-05-26T03:00:34.572248Z | | SST to 10.16.29.103 2023-05-26T03:00:34.572248Z | | SST to garb
2023-05-26T03:40:23.808519Z | | finished sending SST to garb 2023-05-26T03:40:23.808519Z | | finished sending SST to garb
2023-05-26T03:40:23.808574Z | | DESYNCED -> JOINED 2023-05-26T03:40:23.808574Z | | DESYNCED -> JOINED
2023-05-26T03:40:23.808600Z cluster1-2 synced garb | | 2023-05-26T03:40:23.808600Z cluster1-2 synced garb | |
@@ -178,34 +178,34 @@ mysql version 8.0.31 8.
2023-05-26T03:40:26.047565Z cluster1-2 joined | | 2023-05-26T03:40:26.047565Z cluster1-2 joined | |
2023-05-26T03:40:26.047613Z | | cluster1-1 joined 2023-05-26T03:40:26.047613Z | | cluster1-1 joined
2023-05-26T03:40:26.047651Z cluster1-1 joined | | 2023-05-26T03:40:26.047651Z cluster1-1 joined | |
2023-05-26T03:40:26.047664Z | | 10.16.29.103 left 2023-05-26T03:40:26.047664Z | | garb left
2023-05-26T03:40:26.047712Z 10.16.29.103 left | | 2023-05-26T03:40:26.047712Z garb left | |
2023-05-26T03:40:26.047973Z | cluster1-0 joined | 2023-05-26T03:40:26.047973Z | cluster1-0 joined |
2023-05-26T03:40:26.048058Z | cluster1-2 joined | 2023-05-26T03:40:26.048058Z | cluster1-2 joined |
2023-05-26T03:40:26.048117Z | 10.16.29.103 left | 2023-05-26T03:40:26.048117Z | garb left |
2023-05-26T03:40:26.049387Z 10.16.29.103 left | | 2023-05-26T03:40:26.049387Z garb left | |
2023-05-26T03:40:26.049412Z | | 10.16.29.103 left 2023-05-26T03:40:26.049412Z | | garb left
2023-05-26T03:40:26.049454Z PRIMARY(n=3) | | 2023-05-26T03:40:26.049454Z PRIMARY(n=3) | |
2023-05-26T03:40:26.049477Z | | PRIMARY(n=3) 2023-05-26T03:40:26.049477Z | | PRIMARY(n=3)
2023-05-26T03:40:26.049765Z | 10.16.29.103 left | 2023-05-26T03:40:26.049765Z | garb left |
2023-05-26T03:40:26.049947Z | PRIMARY(n=3) | 2023-05-26T03:40:26.049947Z | PRIMARY(n=3) |
2023-05-27T03:00:19.537730Z | cluster1-0 joined | 2023-05-27T03:00:19.537730Z | cluster1-0 joined |
2023-05-27T03:00:19.537767Z | cluster1-2 joined | 2023-05-27T03:00:19.537767Z | cluster1-2 joined |
2023-05-27T03:00:19.537775Z | 10.16.29.248 joined | 2023-05-27T03:00:19.537775Z | garb joined |
2023-05-27T03:00:19.537993Z cluster1-2 joined | | 2023-05-27T03:00:19.537993Z cluster1-2 joined | |
2023-05-27T03:00:19.538086Z 10.16.29.248 joined | | 2023-05-27T03:00:19.538086Z garb joined | |
2023-05-27T03:00:19.538110Z cluster1-1 joined | | 2023-05-27T03:00:19.538110Z cluster1-1 joined | |
2023-05-27T03:00:19.539621Z | PRIMARY(n=4) | 2023-05-27T03:00:19.539621Z | PRIMARY(n=4) |
2023-05-27T03:00:19.539965Z PRIMARY(n=4) | | 2023-05-27T03:00:19.539965Z PRIMARY(n=4) | |
2023-05-27T03:00:19.542417Z | | cluster1-0 joined 2023-05-27T03:00:19.542417Z | | cluster1-0 joined
2023-05-27T03:00:19.542503Z | | 10.16.29.248 joined 2023-05-27T03:00:19.542503Z | | garb joined
2023-05-27T03:00:19.542526Z | | cluster1-1 joined 2023-05-27T03:00:19.542526Z | | cluster1-1 joined
2023-05-27T03:00:19.544132Z | | PRIMARY(n=4) 2023-05-27T03:00:19.544132Z | | PRIMARY(n=4)
2023-05-27T03:00:20.038432Z | cluster1-2 will resync garb | 2023-05-27T03:00:20.038432Z | cluster1-2 will resync garb |
2023-05-27T03:00:20.038598Z cluster1-2 will resync garb | | 2023-05-27T03:00:20.038598Z cluster1-2 will resync garb | |
2023-05-27T03:00:20.043068Z | | local node will resync garb 2023-05-27T03:00:20.043068Z | | local node will resync garb
2023-05-27T03:00:20.043091Z | | SYNCED -> DONOR 2023-05-27T03:00:20.043091Z | | SYNCED -> DONOR
2023-05-27T03:00:31.529154Z | | SST to 10.16.29.248 2023-05-27T03:00:31.529154Z | | SST to garb
2023-05-27T03:39:48.765020Z cluster1-2 synced garb | | 2023-05-27T03:39:48.765020Z cluster1-2 synced garb | |
2023-05-27T03:39:48.765166Z | cluster1-2 synced garb | 2023-05-27T03:39:48.765166Z | cluster1-2 synced garb |
2023-05-27T03:39:48.768602Z | | finished sending SST to garb 2023-05-27T03:39:48.768602Z | | finished sending SST to garb
@@ -213,28 +213,28 @@ mysql version 8.0.31 8.
2023-05-27T03:39:48.769112Z | | JOINED -> SYNCED 2023-05-27T03:39:48.769112Z | | JOINED -> SYNCED
2023-05-27T03:39:51.224587Z cluster1-2 joined | | 2023-05-27T03:39:51.224587Z cluster1-2 joined | |
2023-05-27T03:39:51.224622Z cluster1-1 joined | | 2023-05-27T03:39:51.224622Z cluster1-1 joined | |
2023-05-27T03:39:51.224631Z 10.16.29.248 left | | 2023-05-27T03:39:51.224631Z garb left | |
2023-05-27T03:39:51.224821Z | cluster1-0 joined | 2023-05-27T03:39:51.224821Z | cluster1-0 joined |
2023-05-27T03:39:51.224991Z | cluster1-2 joined | 2023-05-27T03:39:51.224991Z | cluster1-2 joined |
2023-05-27T03:39:51.225009Z | 10.16.29.248 left | 2023-05-27T03:39:51.225009Z | garb left |
2023-05-27T03:39:51.226003Z 10.16.29.248 left | | 2023-05-27T03:39:51.226003Z garb left | |
2023-05-27T03:39:51.226069Z | 10.16.29.248 left | 2023-05-27T03:39:51.226069Z | garb left |
2023-05-27T03:39:51.226111Z | PRIMARY(n=3) | 2023-05-27T03:39:51.226111Z | PRIMARY(n=3) |
2023-05-27T03:39:51.226160Z PRIMARY(n=3) | | 2023-05-27T03:39:51.226160Z PRIMARY(n=3) | |
2023-05-27T03:39:51.228020Z | | cluster1-0 joined 2023-05-27T03:39:51.228020Z | | cluster1-0 joined
2023-05-27T03:39:51.228100Z | | cluster1-1 joined 2023-05-27T03:39:51.228100Z | | cluster1-1 joined
2023-05-27T03:39:51.228115Z | | 10.16.29.248 left 2023-05-27T03:39:51.228115Z | | garb left
2023-05-27T03:39:51.229544Z | | 10.16.29.248 left 2023-05-27T03:39:51.229544Z | | garb left
2023-05-27T03:39:51.229698Z | | PRIMARY(n=3) 2023-05-27T03:39:51.229698Z | | PRIMARY(n=3)
2023-05-28T03:00:20.179665Z cluster1-2 joined | | 2023-05-28T03:00:20.179665Z cluster1-2 joined | |
2023-05-28T03:00:20.179723Z cluster1-1 joined | | 2023-05-28T03:00:20.179723Z cluster1-1 joined | |
2023-05-28T03:00:20.179744Z 10.16.29.190 joined | | 2023-05-28T03:00:20.179744Z garb joined | |
2023-05-28T03:00:20.180516Z | cluster1-0 joined | 2023-05-28T03:00:20.180516Z | cluster1-0 joined |
2023-05-28T03:00:20.180583Z | cluster1-2 joined | 2023-05-28T03:00:20.180583Z | cluster1-2 joined |
2023-05-28T03:00:20.180597Z | 10.16.29.190 joined | 2023-05-28T03:00:20.180597Z | garb joined |
2023-05-28T03:00:20.180757Z | | cluster1-0 joined 2023-05-28T03:00:20.180757Z | | cluster1-0 joined
2023-05-28T03:00:20.180839Z | | cluster1-1 joined 2023-05-28T03:00:20.180839Z | | cluster1-1 joined
2023-05-28T03:00:20.180863Z | | 10.16.29.190 joined 2023-05-28T03:00:20.180863Z | | garb joined
2023-05-28T03:00:20.181677Z PRIMARY(n=4) | | 2023-05-28T03:00:20.181677Z PRIMARY(n=4) | |
2023-05-28T03:00:20.182305Z | PRIMARY(n=4) | 2023-05-28T03:00:20.182305Z | PRIMARY(n=4) |
2023-05-28T03:00:20.182540Z | | PRIMARY(n=4) 2023-05-28T03:00:20.182540Z | | PRIMARY(n=4)
@@ -242,7 +242,7 @@ mysql version 8.0.31 8.
2023-05-28T03:00:20.681257Z | cluster1-2 will resync garb | 2023-05-28T03:00:20.681257Z | cluster1-2 will resync garb |
2023-05-28T03:00:20.681447Z | | local node will resync garb 2023-05-28T03:00:20.681447Z | | local node will resync garb
2023-05-28T03:00:20.681471Z | | SYNCED -> DONOR 2023-05-28T03:00:20.681471Z | | SYNCED -> DONOR
2023-05-28T03:00:32.184853Z | | SST to 10.16.29.190 2023-05-28T03:00:32.184853Z | | SST to garb
2023-05-28T03:40:51.321660Z | | finished sending SST to garb 2023-05-28T03:40:51.321660Z | | finished sending SST to garb
2023-05-28T03:40:51.321733Z | | DESYNCED -> JOINED 2023-05-28T03:40:51.321733Z | | DESYNCED -> JOINED
2023-05-28T03:40:51.321783Z cluster1-2 synced garb | | 2023-05-28T03:40:51.321783Z cluster1-2 synced garb | |
@@ -250,18 +250,18 @@ mysql version 8.0.31 8.
2023-05-28T03:40:51.322593Z | | JOINED -> SYNCED 2023-05-28T03:40:51.322593Z | | JOINED -> SYNCED
2023-05-28T03:40:53.656940Z | | cluster1-0 joined 2023-05-28T03:40:53.656940Z | | cluster1-0 joined
2023-05-28T03:40:53.656966Z | | cluster1-1 joined 2023-05-28T03:40:53.656966Z | | cluster1-1 joined
2023-05-28T03:40:53.656981Z | | 10.16.29.190 left 2023-05-28T03:40:53.656981Z | | garb left
2023-05-28T03:40:53.657404Z | cluster1-0 joined | 2023-05-28T03:40:53.657404Z | cluster1-0 joined |
2023-05-28T03:40:53.657428Z | cluster1-2 joined | 2023-05-28T03:40:53.657428Z | cluster1-2 joined |
2023-05-28T03:40:53.657443Z | 10.16.29.190 left | 2023-05-28T03:40:53.657443Z | garb left |
2023-05-28T03:40:53.657819Z cluster1-2 joined | | 2023-05-28T03:40:53.657819Z cluster1-2 joined | |
2023-05-28T03:40:53.657840Z cluster1-1 joined | | 2023-05-28T03:40:53.657840Z cluster1-1 joined | |
2023-05-28T03:40:53.657847Z 10.16.29.190 left | | 2023-05-28T03:40:53.657847Z garb left | |
2023-05-28T03:40:53.658835Z 10.16.29.190 left | | 2023-05-28T03:40:53.658835Z garb left | |
2023-05-28T03:40:53.658995Z | | 10.16.29.190 left 2023-05-28T03:40:53.658995Z | | garb left
2023-05-28T03:40:53.659027Z PRIMARY(n=3) | | 2023-05-28T03:40:53.659027Z PRIMARY(n=3) | |
2023-05-28T03:40:53.659156Z | | PRIMARY(n=3) 2023-05-28T03:40:53.659156Z | | PRIMARY(n=3)
2023-05-28T03:40:53.659348Z | 10.16.29.190 left | 2023-05-28T03:40:53.659348Z | garb left |
2023-05-28T03:40:53.659384Z | PRIMARY(n=3) | 2023-05-28T03:40:53.659384Z | PRIMARY(n=3) |
2023-05-28T08:09:01.368781Z | received shutdown | 2023-05-28T08:09:01.368781Z | received shutdown |
2023-05-28T08:19:06.122140Z cluster1-1 suspected to be down | | 2023-05-28T08:19:06.122140Z cluster1-1 suspected to be down | |
@@ -459,15 +459,15 @@ mysql version 8.0.31 8.
2023-05-29T05:00:37.738615Z | | garb joined 2023-05-29T05:00:37.738615Z | | garb joined
2023-05-29T05:00:37.738711Z | | cluster1-0 joined 2023-05-29T05:00:37.738711Z | | cluster1-0 joined
2023-05-29T05:00:37.738735Z | | cluster1-1 joined 2023-05-29T05:00:37.738735Z | | cluster1-1 joined
2023-05-29T05:00:37.738755Z | | 10.16.29.220 joined 2023-05-29T05:00:37.738755Z | | garb joined
2023-05-29T05:00:37.740237Z garb joined | | 2023-05-29T05:00:37.740237Z garb joined | |
2023-05-29T05:00:37.740269Z | garb joined | 2023-05-29T05:00:37.740269Z | garb joined |
2023-05-29T05:00:37.740320Z cluster1-1 joined | | 2023-05-29T05:00:37.740320Z cluster1-1 joined | |
2023-05-29T05:00:37.740345Z cluster1-2 joined | | 2023-05-29T05:00:37.740345Z cluster1-2 joined | |
2023-05-29T05:00:37.740366Z 10.16.29.220 joined | | 2023-05-29T05:00:37.740366Z garb joined | |
2023-05-29T05:00:37.740402Z | cluster1-0 joined | 2023-05-29T05:00:37.740402Z | cluster1-0 joined |
2023-05-29T05:00:37.740430Z | cluster1-2 joined | 2023-05-29T05:00:37.740430Z | cluster1-2 joined |
2023-05-29T05:00:37.740475Z | 10.16.29.220 joined | 2023-05-29T05:00:37.740475Z | garb joined |
2023-05-29T05:00:37.740651Z | | PRIMARY(n=5) 2023-05-29T05:00:37.740651Z | | PRIMARY(n=5)
2023-05-29T05:00:37.742138Z | PRIMARY(n=5) | 2023-05-29T05:00:37.742138Z | PRIMARY(n=5) |
2023-05-29T05:00:37.742177Z PRIMARY(n=5) | | 2023-05-29T05:00:37.742177Z PRIMARY(n=5) | |
@@ -475,7 +475,7 @@ mysql version 8.0.31 8.
2023-05-29T05:00:38.239874Z | | SYNCED -> DONOR 2023-05-29T05:00:38.239874Z | | SYNCED -> DONOR
2023-05-29T05:00:38.241430Z | cluster1-2 will resync garb | 2023-05-29T05:00:38.241430Z | cluster1-2 will resync garb |
2023-05-29T05:00:38.241476Z cluster1-2 will resync garb | | 2023-05-29T05:00:38.241476Z cluster1-2 will resync garb | |
2023-05-29T05:00:49.731756Z | | SST to 10.16.29.220 2023-05-29T05:00:49.731756Z | | SST to garb
2023-05-29T05:54:56.049218Z | | finished sending SST to garb 2023-05-29T05:54:56.049218Z | | finished sending SST to garb
2023-05-29T05:54:56.049291Z | | DESYNCED -> JOINED 2023-05-29T05:54:56.049291Z | | DESYNCED -> JOINED
2023-05-29T05:54:56.049762Z | | JOINED -> SYNCED 2023-05-29T05:54:56.049762Z | | JOINED -> SYNCED
@@ -484,15 +484,15 @@ mysql version 8.0.31 8.
2023-05-29T05:54:58.219414Z | | garb joined 2023-05-29T05:54:58.219414Z | | garb joined
2023-05-29T05:54:58.219474Z | | cluster1-0 joined 2023-05-29T05:54:58.219474Z | | cluster1-0 joined
2023-05-29T05:54:58.219489Z | | cluster1-1 joined 2023-05-29T05:54:58.219489Z | | cluster1-1 joined
2023-05-29T05:54:58.219504Z | | 10.16.29.220 left 2023-05-29T05:54:58.219504Z | | garb left
2023-05-29T05:54:58.220860Z garb joined | | 2023-05-29T05:54:58.220860Z garb joined | |
2023-05-29T05:54:58.220913Z cluster1-1 joined | | 2023-05-29T05:54:58.220913Z cluster1-1 joined | |
2023-05-29T05:54:58.220925Z cluster1-2 joined | | 2023-05-29T05:54:58.220925Z cluster1-2 joined | |
2023-05-29T05:54:58.220954Z 10.16.29.220 left | | 2023-05-29T05:54:58.220954Z garb left | |
2023-05-29T05:54:58.221144Z | garb joined | 2023-05-29T05:54:58.221144Z | garb joined |
2023-05-29T05:54:58.221241Z | cluster1-0 joined | 2023-05-29T05:54:58.221241Z | cluster1-0 joined |
2023-05-29T05:54:58.221263Z | cluster1-2 joined | 2023-05-29T05:54:58.221263Z | cluster1-2 joined |
2023-05-29T05:54:58.221280Z | 10.16.29.220 left | 2023-05-29T05:54:58.221280Z | garb left |
2023-05-29T05:55:03.220834Z | | garb suspected to be down 2023-05-29T05:55:03.220834Z | | garb suspected to be down
2023-05-29T05:55:03.222309Z garb suspected to be down | | 2023-05-29T05:55:03.222309Z garb suspected to be down | |
2023-05-29T05:55:03.722761Z | garb suspected to be down | 2023-05-29T05:55:03.722761Z | garb suspected to be down |
@@ -510,13 +510,13 @@ mysql version 8.0.31 8.
2023-05-29T05:55:04.225876Z PRIMARY(n=3) | | 2023-05-29T05:55:04.225876Z PRIMARY(n=3) | |
2023-05-29T06:18:41.339691Z | | cluster1-0 joined 2023-05-29T06:18:41.339691Z | | cluster1-0 joined
2023-05-29T06:18:41.339777Z | | cluster1-1 joined 2023-05-29T06:18:41.339777Z | | cluster1-1 joined
2023-05-29T06:18:41.339802Z | | 10.16.27.183 joined 2023-05-29T06:18:41.339802Z | | garb joined
2023-05-29T06:18:41.340638Z cluster1-1 joined | | 2023-05-29T06:18:41.340638Z cluster1-1 joined | |
2023-05-29T06:18:41.340748Z cluster1-2 joined | | 2023-05-29T06:18:41.340748Z cluster1-2 joined | |
2023-05-29T06:18:41.340777Z 10.16.27.183 joined | | 2023-05-29T06:18:41.340777Z garb joined | |
2023-05-29T06:18:41.340846Z | cluster1-0 joined | 2023-05-29T06:18:41.340846Z | cluster1-0 joined |
2023-05-29T06:18:41.340903Z | cluster1-2 joined | 2023-05-29T06:18:41.340903Z | cluster1-2 joined |
2023-05-29T06:18:41.340914Z | 10.16.27.183 joined | 2023-05-29T06:18:41.340914Z | garb joined |
2023-05-29T06:18:41.341398Z | | PRIMARY(n=4) 2023-05-29T06:18:41.341398Z | | PRIMARY(n=4)
2023-05-29T06:18:41.342417Z | PRIMARY(n=4) | 2023-05-29T06:18:41.342417Z | PRIMARY(n=4) |
2023-05-29T06:18:41.342618Z PRIMARY(n=4) | | 2023-05-29T06:18:41.342618Z PRIMARY(n=4) | |
@@ -524,19 +524,19 @@ mysql version 8.0.31 8.
2023-05-29T06:18:41.840393Z | | SYNCED -> DONOR 2023-05-29T06:18:41.840393Z | | SYNCED -> DONOR
2023-05-29T06:18:41.841468Z | cluster1-2 will resync garb | 2023-05-29T06:18:41.841468Z | cluster1-2 will resync garb |
2023-05-29T06:18:41.841677Z cluster1-2 will resync garb | | 2023-05-29T06:18:41.841677Z cluster1-2 will resync garb | |
2023-05-29T06:18:53.370532Z | | SST to 10.16.27.183 2023-05-29T06:18:53.370532Z | | SST to garb
2023-05-29T06:21:16.932469Z | | garb joined 2023-05-29T06:21:16.932469Z | | garb joined
2023-05-29T06:21:16.932542Z | | cluster1-0 joined 2023-05-29T06:21:16.932542Z | | cluster1-0 joined
2023-05-29T06:21:16.932559Z | | cluster1-1 joined 2023-05-29T06:21:16.932559Z | | cluster1-1 joined
2023-05-29T06:21:16.932574Z | | 10.16.27.183 joined 2023-05-29T06:21:16.932574Z | | garb joined
2023-05-29T06:21:16.933460Z | garb joined | 2023-05-29T06:21:16.933460Z | garb joined |
2023-05-29T06:21:16.933530Z | cluster1-0 joined | 2023-05-29T06:21:16.933530Z | cluster1-0 joined |
2023-05-29T06:21:16.933543Z | cluster1-2 joined | 2023-05-29T06:21:16.933543Z | cluster1-2 joined |
2023-05-29T06:21:16.933554Z | 10.16.27.183 joined | 2023-05-29T06:21:16.933554Z | garb joined |
2023-05-29T06:21:16.933921Z garb joined | | 2023-05-29T06:21:16.933921Z garb joined | |
2023-05-29T06:21:16.933996Z cluster1-1 joined | | 2023-05-29T06:21:16.933996Z cluster1-1 joined | |
2023-05-29T06:21:16.934011Z cluster1-2 joined | | 2023-05-29T06:21:16.934011Z cluster1-2 joined | |
2023-05-29T06:21:16.934022Z 10.16.27.183 joined | | 2023-05-29T06:21:16.934022Z garb joined | |
2023-05-29T06:21:16.934118Z | | PRIMARY(n=5) 2023-05-29T06:21:16.934118Z | | PRIMARY(n=5)
2023-05-29T06:21:16.935175Z | PRIMARY(n=5) | 2023-05-29T06:21:16.935175Z | PRIMARY(n=5) |
2023-05-29T06:21:16.935415Z PRIMARY(n=5) | | 2023-05-29T06:21:16.935415Z PRIMARY(n=5) | |
@@ -553,44 +553,44 @@ mysql version 8.0.31 8.
2023-05-29T07:16:33.678136Z | | garb joined 2023-05-29T07:16:33.678136Z | | garb joined
2023-05-29T07:16:33.678217Z | | cluster1-0 joined 2023-05-29T07:16:33.678217Z | | cluster1-0 joined
2023-05-29T07:16:33.678236Z | | cluster1-1 joined 2023-05-29T07:16:33.678236Z | | cluster1-1 joined
2023-05-29T07:16:33.678252Z | | 10.16.27.183 left 2023-05-29T07:16:33.678252Z | | garb left
2023-05-29T07:16:34.694046Z | SST error | 2023-05-29T07:16:34.694046Z | SST error |
2023-05-29T07:16:38.678115Z garb suspected to be down | | 2023-05-29T07:16:38.678115Z garb suspected to be down | |
2023-05-29T07:16:38.678140Z cluster1-2 suspected to be down | | 2023-05-29T07:16:38.678140Z cluster1-2 suspected to be down | |
2023-05-29T07:16:38.678174Z | garb suspected to be down | 2023-05-29T07:16:38.678174Z | garb suspected to be down |
2023-05-29T07:16:38.678187Z 10.16.27.183 suspected to be down | | 2023-05-29T07:16:38.678187Z garb suspected to be down | |
2023-05-29T07:16:38.678199Z | cluster1-2 suspected to be down | 2023-05-29T07:16:38.678199Z | cluster1-2 suspected to be down |
2023-05-29T07:16:38.678232Z | 10.16.27.183 suspected to be down | 2023-05-29T07:16:38.678232Z | garb suspected to be down |
2023-05-29T07:16:38.679876Z | | garb suspected to be down 2023-05-29T07:16:38.679876Z | | garb suspected to be down
2023-05-29T07:16:38.679923Z | | cluster1-0 suspected to be down 2023-05-29T07:16:38.679923Z | | cluster1-0 suspected to be down
2023-05-29T07:16:38.679965Z | | cluster1-1 suspected to be down 2023-05-29T07:16:38.679965Z | | cluster1-1 suspected to be down
2023-05-29T07:16:39.178309Z garb suspected to be down | | 2023-05-29T07:16:39.178309Z garb suspected to be down | |
2023-05-29T07:16:39.178383Z | garb suspected to be down | 2023-05-29T07:16:39.178383Z | garb suspected to be down |
2023-05-29T07:16:39.178407Z cluster1-2 suspected to be down | | 2023-05-29T07:16:39.178407Z cluster1-2 suspected to be down | |
2023-05-29T07:16:39.178437Z 10.16.27.183 suspected to be down | | 2023-05-29T07:16:39.178437Z garb suspected to be down | |
2023-05-29T07:16:39.178468Z | cluster1-2 suspected to be down | 2023-05-29T07:16:39.178468Z | cluster1-2 suspected to be down |
2023-05-29T07:16:39.178496Z | 10.16.27.183 suspected to be down | 2023-05-29T07:16:39.178496Z | garb suspected to be down |
2023-05-29T07:16:39.678585Z | garb suspected to be down | 2023-05-29T07:16:39.678585Z | garb suspected to be down |
2023-05-29T07:16:39.678620Z garb suspected to be down | | 2023-05-29T07:16:39.678620Z garb suspected to be down | |
2023-05-29T07:16:39.678649Z | cluster1-2 suspected to be down | 2023-05-29T07:16:39.678649Z | cluster1-2 suspected to be down |
2023-05-29T07:16:39.678672Z | 10.16.27.183 suspected to be down | 2023-05-29T07:16:39.678672Z | garb suspected to be down |
2023-05-29T07:16:39.678717Z cluster1-2 suspected to be down | | 2023-05-29T07:16:39.678717Z cluster1-2 suspected to be down | |
2023-05-29T07:16:39.678763Z 10.16.27.183 suspected to be down | | 2023-05-29T07:16:39.678763Z garb suspected to be down | |
2023-05-29T07:16:39.680986Z | | NON-PRIMARY(n=1) 2023-05-29T07:16:39.680986Z | | NON-PRIMARY(n=1)
2023-05-29T07:16:39.681204Z | | SYNCED -> OPEN 2023-05-29T07:16:39.681204Z | | SYNCED -> OPEN
2023-05-29T07:16:39.681241Z | | NON-PRIMARY(n=1) 2023-05-29T07:16:39.681241Z | | NON-PRIMARY(n=1)
2023-05-29T07:16:40.178884Z garb suspected to be down | | 2023-05-29T07:16:40.178884Z garb suspected to be down | |
2023-05-29T07:16:40.178983Z cluster1-2 suspected to be down | | 2023-05-29T07:16:40.178983Z cluster1-2 suspected to be down | |
2023-05-29T07:16:40.179015Z 10.16.27.183 suspected to be down | | 2023-05-29T07:16:40.179015Z garb suspected to be down | |
2023-05-29T07:16:40.179037Z | garb suspected to be down | 2023-05-29T07:16:40.179037Z | garb suspected to be down |
2023-05-29T07:16:40.179126Z | cluster1-2 suspected to be down | 2023-05-29T07:16:40.179126Z | cluster1-2 suspected to be down |
2023-05-29T07:16:40.179154Z | 10.16.27.183 suspected to be down | 2023-05-29T07:16:40.179154Z | garb suspected to be down |
2023-05-29T07:16:40.679187Z garb suspected to be down | | 2023-05-29T07:16:40.679187Z garb suspected to be down | |
2023-05-29T07:16:40.679234Z | garb suspected to be down | 2023-05-29T07:16:40.679234Z | garb suspected to be down |
2023-05-29T07:16:40.679286Z | cluster1-2 suspected to be down | 2023-05-29T07:16:40.679286Z | cluster1-2 suspected to be down |
2023-05-29T07:16:40.679287Z cluster1-2 suspected to be down | | 2023-05-29T07:16:40.679287Z cluster1-2 suspected to be down | |
2023-05-29T07:16:40.679299Z | 10.16.27.183 suspected to be down | 2023-05-29T07:16:40.679299Z | garb suspected to be down |
2023-05-29T07:16:40.679313Z 10.16.27.183 suspected to be down | | 2023-05-29T07:16:40.679313Z garb suspected to be down | |
2023-05-29T07:16:41.679431Z garb suspected to be down | | 2023-05-29T07:16:41.679431Z garb suspected to be down | |
2023-05-29T07:16:41.679447Z cluster1-2 suspected to be down | | 2023-05-29T07:16:41.679447Z cluster1-2 suspected to be down | |
2023-05-29T07:16:41.679615Z | garb suspected to be down | 2023-05-29T07:16:41.679615Z | garb suspected to be down |
@@ -647,11 +647,11 @@ mysql version 8.0.31 8.
2023-05-29T07:16:48.182140Z cluster1-2 suspected to be down | | 2023-05-29T07:16:48.182140Z cluster1-2 suspected to be down | |
2023-05-29T07:16:48.183482Z | garb suspected to be down | 2023-05-29T07:16:48.183482Z | garb suspected to be down |
2023-05-29T07:16:48.183563Z | cluster1-2 suspected to be down | 2023-05-29T07:16:48.183563Z | cluster1-2 suspected to be down |
2023-05-29T07:16:48.682756Z | 10.16.27.183 left | 2023-05-29T07:16:48.682756Z | garb left |
2023-05-29T07:16:48.682898Z | NON-PRIMARY(n=1) | 2023-05-29T07:16:48.682898Z | NON-PRIMARY(n=1) |
2023-05-29T07:16:48.682999Z | DESYNCED -> OPEN | 2023-05-29T07:16:48.682999Z | DESYNCED -> OPEN |
2023-05-29T07:16:48.683014Z | NON-PRIMARY(n=1) | 2023-05-29T07:16:48.683014Z | NON-PRIMARY(n=1) |
2023-05-29T07:16:48.683734Z 10.16.27.183 left | | 2023-05-29T07:16:48.683734Z garb left | |
2023-05-29T07:16:48.683956Z NON-PRIMARY(n=1) | | 2023-05-29T07:16:48.683956Z NON-PRIMARY(n=1) | |
2023-05-29T07:16:48.684193Z SYNCED -> OPEN | | 2023-05-29T07:16:48.684193Z SYNCED -> OPEN | |
2023-05-29T07:16:48.684230Z NON-PRIMARY(n=1) | | 2023-05-29T07:16:48.684230Z NON-PRIMARY(n=1) | |