From 117d683872a2ad42d950371756bbce1b973f4a70 Mon Sep 17 00:00:00 2001 From: Yoann La Cancellera Date: Wed, 11 Oct 2023 15:50:49 +0200 Subject: [PATCH] Change: simplify verbose mode --- .../display/timelinecli.go | 2 +- src/go/pt-galera-log-explainer/main.go | 2 +- src/go/pt-galera-log-explainer/regex/sst.go | 2 - src/go/pt-galera-log-explainer/regex/views.go | 3 - .../expected/merge_rotated_daily_list_all | 5 + ...d_daily_list_all_since_keeping_latest_logs | 10 + .../tests/expected/upgrade_list_all | 13 + .../tests/expected/upgrade_list_all_no_color | 13 + .../tests/expected/upgrade_list_all_since | 3 + .../expected/upgrade_list_all_since_until | 1 + .../tests/expected/upgrade_list_all_until | 455 +++++++++--------- .../upgrade_list_all_until_hiding_1_node | 418 ++++++++-------- .../upgrade_list_all_until_hiding_2_nodes | 272 +++++------ .../tests/expected/upgrade_list_sst | 11 + .../tests/expected/upgrade_list_views | 280 +++++------ .../pt-galera-log-explainer/types/loginfo.go | 2 - 16 files changed, 784 insertions(+), 708 deletions(-) diff --git a/src/go/pt-galera-log-explainer/display/timelinecli.go b/src/go/pt-galera-log-explainer/display/timelinecli.go index 6d6f92c1..fe1963ca 100644 --- a/src/go/pt-galera-log-explainer/display/timelinecli.go +++ b/src/go/pt-galera-log-explainer/display/timelinecli.go @@ -71,7 +71,7 @@ func TimelineCLI(timeline types.Timeline, verbosity types.Verbosity) { timeline.Dequeue(node) msg := loginfo.Msg(latestContext[node]) - if verbosity > loginfo.Verbosity && msg != "" { + if verbosity >= loginfo.Verbosity && msg != "" { args = append(args, msg) displayedValue++ } else { diff --git a/src/go/pt-galera-log-explainer/main.go b/src/go/pt-galera-log-explainer/main.go index b09ccfbe..32c1d4a4 100644 --- a/src/go/pt-galera-log-explainer/main.go +++ b/src/go/pt-galera-log-explainer/main.go @@ -30,7 +30,7 @@ var CLI struct { NoColor bool Since *time.Time `help:"Only list events after this date, format: 2023-01-23T03:53:40Z (RFC3339)"` Until *time.Time `help:"Only list events before this date"` - Verbosity types.Verbosity `type:"counter" short:"v" default:"1" help:"-v: Detailed (default), -vv: DebugMySQL (add every mysql info the tool used), -vvv: Debug (internal tool debug)"` + Verbosity types.Verbosity `type:"counter" short:"v" default:"0" help:"-v: DebugMySQL (add every mysql info the tool used), -vv: Debug (internal tool debug)"` PxcOperator bool `default:"false" help:"Analyze logs from Percona PXC operator. Off by default because it negatively impacts performance for non-k8s setups"` ExcludeRegexes []string `help:"Remove regexes from analysis. List regexes using 'pt-galera-log-explainer regex-list'"` MergeByDirectory bool `help:"Instead of relying on identification, merge contexts and columns by base directory. Very useful when dealing with many small logs organized per directories."` diff --git a/src/go/pt-galera-log-explainer/regex/sst.go b/src/go/pt-galera-log-explainer/regex/sst.go index 6e57e0bb..1884d457 100644 --- a/src/go/pt-galera-log-explainer/regex/sst.go +++ b/src/go/pt-galera-log-explainer/regex/sst.go @@ -38,7 +38,6 @@ var SSTMap = types.RegexMap{ return donor + utils.Paint(utils.GreenText, " will resync ") + joiner } }, - Verbosity: types.Detailed, }, "RegexSSTResourceUnavailable": &types.LogRegex{ @@ -253,7 +252,6 @@ var SSTMap = types.RegexMap{ Handler: func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) { return ctx, types.SimpleDisplayer("preparing SST backup") }, - Verbosity: types.Detailed, }, "RegexTimeoutReceivingFirstData": &types.LogRegex{ diff --git a/src/go/pt-galera-log-explainer/regex/views.go b/src/go/pt-galera-log-explainer/regex/views.go index 692db875..d22f4f19 100644 --- a/src/go/pt-galera-log-explainer/regex/views.go +++ b/src/go/pt-galera-log-explainer/regex/views.go @@ -103,7 +103,6 @@ var ViewsMap = types.RegexMap{ } return ctx, types.SimpleDisplayer(hash + utils.Paint(utils.YellowText, " suspected to be down")) }, - Verbosity: types.Detailed, }, "RegexNodeChangedIdentity": &types.LogRegex{ @@ -128,7 +127,6 @@ var ViewsMap = types.RegexMap{ return types.DisplayNodeSimplestForm(ctx, ip) + utils.Paint(utils.YellowText, " changed identity") } }, - Verbosity: types.Detailed, }, "RegexWsrepUnsafeBootstrap": &types.LogRegex{ @@ -172,7 +170,6 @@ var ViewsMap = types.RegexMap{ Handler: func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) { return ctx, types.SimpleDisplayer(utils.Paint(utils.YellowText, "no grastate.dat file")) }, - Verbosity: types.Detailed, }, "RegexBootstrapingDefaultState": &types.LogRegex{ Regex: regexp.MustCompile("Bootstraping with default state"), diff --git a/src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all b/src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all index 7adaf650..a3b16922 100644 --- a/src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all +++ b/src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all @@ -11,6 +11,7 @@ mysql version 8.0.28 2023-03-15T20:20:12.714291+02:00 node2 joined | | 2023-03-15T20:20:12.714331+02:00 node3 joined | | 2023-03-15T20:20:13.776977+02:00 PRIMARY(n=3) | | +2023-03-15T20:20:14.839684+02:00 local node will resync node3 | | 2023-03-15T20:20:14.839723+02:00 SYNCED -> DONOR | | 2023-03-15T20:20:15.799020+02:00 IST will be used | | 2023-03-15T20:20:16.850525+02:00 finished sending IST to node3 | | @@ -710,9 +711,13 @@ mysql version 8.0.28 2023-03-18T21:25:07.630920+02:00 PRIMARY(n=3) | | 2023-03-18T21:25:07.646531+02:00 OPEN -> PRIMARY | | 2023-03-18T21:25:08.577339+02:00 will receive SST | | +2023-03-18T21:25:08.586102+02:00 |  local node will resync node1 | 2023-03-18T21:25:08.586160+02:00 |  SYNCED -> DONOR | +2023-03-18T21:25:08.588074+02:00 node2 will resync local node |  | 2023-03-18T21:25:08.588096+02:00 PRIMARY -> JOINER |  | +2023-03-18T21:25:08.588498+02:00 |  |  node2 will resync node1 2023-03-18T21:25:10.414098+02:00 receiving SST |  | +2023-03-18T21:40:25.127116+02:00 preparing SST backup |  | 2023-03-18T21:40:25.130185+02:00 |  |  node2 synced node1 2023-03-18T21:40:25.133527+02:00 |  finished sending SST to node1 | 2023-03-18T21:40:25.133593+02:00 |  DESYNCED -> JOINED | diff --git a/src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_since_keeping_latest_logs b/src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_since_keeping_latest_logs index e9a438db..e16122e1 100644 --- a/src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_since_keeping_latest_logs +++ b/src/go/pt-galera-log-explainer/tests/expected/merge_rotated_daily_list_all_since_keeping_latest_logs @@ -31,9 +31,13 @@ mysql version 8.0.28 2023-03-18T21:25:07.630920+02:00 PRIMARY(n=3) | | 2023-03-18T21:25:07.646531+02:00 OPEN -> PRIMARY | | 2023-03-18T21:25:08.577339+02:00 will receive SST | | +2023-03-18T21:25:08.586102+02:00 |  local node will resync node1 | 2023-03-18T21:25:08.586160+02:00 |  SYNCED -> DONOR | +2023-03-18T21:25:08.588074+02:00 node2 will resync local node |  | 2023-03-18T21:25:08.588096+02:00 PRIMARY -> JOINER |  | +2023-03-18T21:25:08.588498+02:00 |  |  node2 will resync node1 2023-03-18T21:25:10.414098+02:00 receiving SST |  | +2023-03-18T21:40:25.127116+02:00 preparing SST backup |  | 2023-03-18T21:40:25.130185+02:00 |  |  node2 synced node1 2023-03-18T21:40:25.133527+02:00 |  finished sending SST to node1 | 2023-03-18T21:40:25.133593+02:00 |  DESYNCED -> JOINED | @@ -53,3 +57,9 @@ mysql version 8.0.28 2023-03-18T21:40:39.454059+02:00 |  |  PRIMARY(n=2) 2023-03-19T08:18:47.520541+02:00 received shutdown |  | 2023-03-19T08:19:02.849987+02:00 shutdown complete |  | + +identifier node1 node2 node3 +current path tests/logs/merge_rotated_daily/node1.20230318.log tests/logs/merge_rotated_daily/node2.20230318.log tests/logs/merge_rotated_daily/node3.20230318.log +last known ip 172.17.0.2 +last known name node1 node2 node3 +mysql version 8.0.28 diff --git a/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all b/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all index 1c8be669..e3719f0b 100644 --- a/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all +++ b/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all @@ -16,6 +16,8 @@ mysql version 8.0.28 8.0.28 2023-03-12T07:34:57.286990Z | node1 joined | 2023-03-12T07:34:57.287111Z | node3 left | 2023-03-12T07:34:57.290903Z | node3 left | +2023-03-12T07:35:02.791416Z | (repeated x17)node1 suspected to be down | +2023-03-12T07:35:11.793101Z | node1 suspected to be down | 2023-03-12T07:35:12.293578Z | PRIMARY(n=2) | 2023-03-12T07:35:12.293705Z | NON-PRIMARY(n=1) | 2023-03-12T07:35:12.293723Z | SYNCED -> OPEN | @@ -90,6 +92,7 @@ mysql version 8.0.28 8.0.28 2023-03-12T11:24:33.334761Z | JOINED -> SYNCED | 2023-03-12T11:35:14.693312Z | node3 joined | 2023-03-12T11:35:14.695410Z | PRIMARY(n=2) | +2023-03-12T11:35:16.321586Z | local node will resync node3 | 2023-03-12T11:35:16.321642Z | SYNCED -> DONOR | 2023-03-12T11:35:16.342707Z | IST to node3(seqno:170403898) | 2023-03-12T11:35:17.118100Z | IST will be used | @@ -101,6 +104,7 @@ mysql version 8.0.28 8.0.28 2023-03-12T11:35:21.035794Z | PRIMARY(n=1) | 2023-03-12T11:39:20.681083Z | node3 joined | 2023-03-12T11:39:20.683800Z | PRIMARY(n=2) | +2023-03-12T11:39:21.948501Z | local node will resync node3 | 2023-03-12T11:39:21.948554Z | SYNCED -> DONOR | 2023-03-12T11:39:21.952242Z | IST to node3(seqno:170403900) | 2023-03-12T11:39:33.420743Z | SST to node3 | @@ -136,7 +140,9 @@ mysql version 8.0.28 8.0.28 2023-03-12T12:48:43.820929Z | |  PRIMARY(n=2) 2023-03-12T12:48:43.822001Z | |  OPEN -> PRIMARY 2023-03-12T12:48:44.597299Z | |  will receive IST(seqno:170403905) +2023-03-12T12:48:44.599287Z | local node will resync node3 |  2023-03-12T12:48:44.599341Z | SYNCED -> DONOR |  +2023-03-12T12:48:44.599346Z | |  node2 will resync local node 2023-03-12T12:48:44.599377Z | |  PRIMARY -> JOINER 2023-03-12T12:48:44.616436Z | IST to node3(seqno:170403905) |  2023-03-12T12:48:45.044873Z | IST will be used |  @@ -154,6 +160,8 @@ mysql version 8.0.28 8.0.28 2023-03-12T13:04:24.476863Z | |  node2 joined 2023-03-12T13:04:24.478964Z | PRIMARY(n=3) |  2023-03-12T13:04:24.479206Z | |  PRIMARY(n=3) +2023-03-12T13:04:25.731994Z | node3 will resync node1 |  +2023-03-12T13:04:25.732124Z | |  local node will resync node1 2023-03-12T13:04:25.732132Z | |  SYNCED -> DONOR 2023-03-12T13:04:25.735999Z | |  IST to node1(seqno:170407335) 2023-03-12T13:04:37.415791Z | |  SST to node1 @@ -187,7 +195,9 @@ mysql version 8.0.28 8.0.28 2023-03-12T13:13:12.515641Z | PRIMARY(n=2) |  2023-03-12T13:13:12.516249Z | OPEN -> PRIMARY |  2023-03-12T13:13:13.245723Z | will receive IST(seqno:170407338) |  +2023-03-12T13:13:13.247714Z | node3 will resync local node |  2023-03-12T13:13:13.247750Z | PRIMARY -> JOINER |  +2023-03-12T13:13:13.248015Z | |  local node will resync node2 2023-03-12T13:13:13.248065Z | |  SYNCED -> DONOR 2023-03-12T13:13:13.262238Z | |  IST to node2(seqno:170407338) 2023-03-12T13:13:13.863959Z | |  IST will be used @@ -213,8 +223,11 @@ mysql version 8.0.28 8.0.28 2023-03-12T19:35:06.875717Z PRIMARY(n=3) |  |  2023-03-12T19:35:06.876501Z OPEN -> PRIMARY |  |  2023-03-12T19:35:07.638676Z will receive IST(seqno:178226774) |  |  +2023-03-12T19:35:07.644560Z |  |  local node will resync node1 2023-03-12T19:35:07.644570Z |  |  SYNCED -> DONOR +2023-03-12T19:35:07.644668Z node3 will resync node1 |  |  2023-03-12T19:35:07.644683Z PRIMARY -> JOINER |  |  +2023-03-12T19:35:07.644740Z |  node3 will resync node1 |  2023-03-12T19:36:48.567087Z timeout from donor in gtid/keyring stage |  |  2023-03-12T19:36:48.589084Z SST error |  |  2023-03-12T19:36:48.590054Z |  |  node2 joined diff --git a/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_no_color b/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_no_color index 3d6412fd..f89354c1 100644 --- a/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_no_color +++ b/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_no_color @@ -16,6 +16,8 @@ mysql version 8.0.28 8.0.28 2023-03-12T07:34:57.286990Z | node1 joined | 2023-03-12T07:34:57.287111Z | node3 left | 2023-03-12T07:34:57.290903Z | node3 left | +2023-03-12T07:35:02.791416Z | (repeated x17)node1 suspected to be down | +2023-03-12T07:35:11.793101Z | node1 suspected to be down | 2023-03-12T07:35:12.293578Z | PRIMARY(n=2) | 2023-03-12T07:35:12.293705Z | NON-PRIMARY(n=1) | 2023-03-12T07:35:12.293723Z | SYNCED -> OPEN | @@ -90,6 +92,7 @@ mysql version 8.0.28 8.0.28 2023-03-12T11:24:33.334761Z | JOINED -> SYNCED | 2023-03-12T11:35:14.693312Z | node3 joined | 2023-03-12T11:35:14.695410Z | PRIMARY(n=2) | +2023-03-12T11:35:16.321586Z | local node will resync node3 | 2023-03-12T11:35:16.321642Z | SYNCED -> DONOR | 2023-03-12T11:35:16.342707Z | IST to node3(seqno:170403898) | 2023-03-12T11:35:17.118100Z | IST will be used | @@ -101,6 +104,7 @@ mysql version 8.0.28 8.0.28 2023-03-12T11:35:21.035794Z | PRIMARY(n=1) | 2023-03-12T11:39:20.681083Z | node3 joined | 2023-03-12T11:39:20.683800Z | PRIMARY(n=2) | +2023-03-12T11:39:21.948501Z | local node will resync node3 | 2023-03-12T11:39:21.948554Z | SYNCED -> DONOR | 2023-03-12T11:39:21.952242Z | IST to node3(seqno:170403900) | 2023-03-12T11:39:33.420743Z | SST to node3 | @@ -136,7 +140,9 @@ mysql version 8.0.28 8.0.28 2023-03-12T12:48:43.820929Z | | PRIMARY(n=2) 2023-03-12T12:48:43.822001Z | | OPEN -> PRIMARY 2023-03-12T12:48:44.597299Z | | will receive IST(seqno:170403905) +2023-03-12T12:48:44.599287Z | local node will resync node3 | 2023-03-12T12:48:44.599341Z | SYNCED -> DONOR | +2023-03-12T12:48:44.599346Z | | node2 will resync local node 2023-03-12T12:48:44.599377Z | | PRIMARY -> JOINER 2023-03-12T12:48:44.616436Z | IST to node3(seqno:170403905) | 2023-03-12T12:48:45.044873Z | IST will be used | @@ -154,6 +160,8 @@ mysql version 8.0.28 8.0.28 2023-03-12T13:04:24.476863Z | | node2 joined 2023-03-12T13:04:24.478964Z | PRIMARY(n=3) | 2023-03-12T13:04:24.479206Z | | PRIMARY(n=3) +2023-03-12T13:04:25.731994Z | node3 will resync node1 | +2023-03-12T13:04:25.732124Z | | local node will resync node1 2023-03-12T13:04:25.732132Z | | SYNCED -> DONOR 2023-03-12T13:04:25.735999Z | | IST to node1(seqno:170407335) 2023-03-12T13:04:37.415791Z | | SST to node1 @@ -187,7 +195,9 @@ mysql version 8.0.28 8.0.28 2023-03-12T13:13:12.515641Z | PRIMARY(n=2) | 2023-03-12T13:13:12.516249Z | OPEN -> PRIMARY | 2023-03-12T13:13:13.245723Z | will receive IST(seqno:170407338) | +2023-03-12T13:13:13.247714Z | node3 will resync local node | 2023-03-12T13:13:13.247750Z | PRIMARY -> JOINER | +2023-03-12T13:13:13.248015Z | | local node will resync node2 2023-03-12T13:13:13.248065Z | | SYNCED -> DONOR 2023-03-12T13:13:13.262238Z | | IST to node2(seqno:170407338) 2023-03-12T13:13:13.863959Z | | IST will be used @@ -213,8 +223,11 @@ mysql version 8.0.28 8.0.28 2023-03-12T19:35:06.875717Z PRIMARY(n=3) | | 2023-03-12T19:35:06.876501Z OPEN -> PRIMARY | | 2023-03-12T19:35:07.638676Z will receive IST(seqno:178226774) | | +2023-03-12T19:35:07.644560Z | | local node will resync node1 2023-03-12T19:35:07.644570Z | | SYNCED -> DONOR +2023-03-12T19:35:07.644668Z node3 will resync node1 | | 2023-03-12T19:35:07.644683Z PRIMARY -> JOINER | | +2023-03-12T19:35:07.644740Z | node3 will resync node1 | 2023-03-12T19:36:48.567087Z timeout from donor in gtid/keyring stage | | 2023-03-12T19:36:48.589084Z SST error | | 2023-03-12T19:36:48.590054Z | | node2 joined diff --git a/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_since b/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_since index 21b96faf..7e6edb26 100644 --- a/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_since +++ b/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_since @@ -26,8 +26,11 @@ mysql version 8.0.28 8.0.28 2023-03-12T19:35:06.875717Z PRIMARY(n=3) |  |  2023-03-12T19:35:06.876501Z OPEN -> PRIMARY |  |  2023-03-12T19:35:07.638676Z will receive IST(seqno:178226774) |  |  +2023-03-12T19:35:07.644560Z |  |  local node will resync node1 2023-03-12T19:35:07.644570Z |  |  SYNCED -> DONOR +2023-03-12T19:35:07.644668Z node3 will resync node1 |  |  2023-03-12T19:35:07.644683Z PRIMARY -> JOINER |  |  +2023-03-12T19:35:07.644740Z |  node3 will resync node1 |  2023-03-12T19:36:48.567087Z timeout from donor in gtid/keyring stage |  |  2023-03-12T19:36:48.589084Z SST error |  |  2023-03-12T19:36:48.590054Z |  |  node2 joined diff --git a/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_since_until b/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_since_until index d199a514..554a3f2d 100644 --- a/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_since_until +++ b/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_since_until @@ -26,4 +26,5 @@ mysql version 8.0.28 2023-03-12T19:35:06.875717Z PRIMARY(n=3) |  |  2023-03-12T19:35:06.876501Z OPEN -> PRIMARY |  |  2023-03-12T19:35:07.638676Z will receive IST(seqno:178226774) |  |  +2023-03-12T19:35:07.644560Z |  |  local node will resync node1 2023-03-12T19:35:07.644570Z |  |  SYNCED -> DONOR diff --git a/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_until b/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_until index 16287212..08c76354 100644 --- a/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_until +++ b/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_until @@ -1,222 +1,233 @@ -identifier 172.17.0.2 node2 node3 -current path tests/logs/upgrade/node1.log tests/logs/upgrade/node2.log tests/logs/upgrade/node3.log -last known ip 172.17.0.2 172.17.0.3 172.17.0.4 -last known name node2 node3 -mysql version 8.0.28 8.0.28 8.0.28 - -2023-03-12T07:24:13.733958Z | starting(5.7.40) | -2023-03-12T07:24:13.771126Z | started(cluster) | -2023-03-12T07:24:14.289375Z | node1 joined | -2023-03-12T07:24:14.289412Z | node3 joined | -2023-03-12T07:24:14.789002Z | CLOSED -> OPEN | -2023-03-12T07:24:14.789075Z | PRIMARY(n=3) | -2023-03-12T07:24:14.789560Z | (restored)OPEN -> JOINED | -2023-03-12T07:24:14.789785Z | JOINED -> SYNCED | -2023-03-12T07:34:47.289292Z | received shutdown | -2023-03-12T07:34:57.286990Z | node1 joined | -2023-03-12T07:34:57.287111Z | node3 left | -2023-03-12T07:34:57.290903Z | node3 left | -2023-03-12T07:35:12.293578Z | PRIMARY(n=2) | -2023-03-12T07:35:12.293705Z | NON-PRIMARY(n=1) | -2023-03-12T07:35:12.293723Z | SYNCED -> OPEN | -2023-03-12T07:35:12.293760Z | OPEN -> CLOSED | -2023-03-12T07:35:18.533851Z | shutdown complete | -2023-03-12T07:38:06.673334Z | starting(5.7.40) | -2023-03-12T07:38:06.680025Z | started(cluster) | -2023-03-12T07:38:06.681065Z | safe_to_bootstrap: 1 | -2023-03-12T07:38:06.693619Z | bootstrapping | -2023-03-12T07:38:06.695987Z | CLOSED -> OPEN | -2023-03-12T07:38:06.696042Z | PRIMARY(n=1) | -2023-03-12T07:38:06.696187Z | (restored)OPEN -> JOINED | -2023-03-12T07:38:06.696210Z | JOINED -> SYNCED | -2023-03-12T07:39:27.162350Z | node3 joined | -2023-03-12T07:39:27.164824Z | PRIMARY(n=2) | -2023-03-12T07:43:09.063375Z | node1 joined | -2023-03-12T07:43:09.063430Z | node3 joined | -2023-03-12T07:43:09.065740Z | PRIMARY(n=3) | -2023-03-12T07:49:45.317891Z | received shutdown | -2023-03-12T07:49:55.319157Z | NON-PRIMARY(n=1) | -2023-03-12T07:49:55.319203Z | SYNCED -> OPEN | -2023-03-12T07:49:55.319230Z | OPEN -> CLOSED | -2023-03-12T07:50:00.605309Z | shutdown complete | -2023-03-12T08:46:48.943442Z | starting(5.7.40) | -2023-03-12T08:46:48.947933Z | started(cluster) | -2023-03-12T08:46:48.992365Z | node1 joined | -2023-03-12T08:46:49.463255Z | CLOSED -> OPEN | -2023-03-12T08:46:49.463334Z | PRIMARY(n=2) | -2023-03-12T08:46:49.463988Z | (restored)OPEN -> JOINED | -2023-03-12T08:46:49.464124Z | JOINED -> SYNCED | -2023-03-12T08:48:28.470198Z | node1 left | -2023-03-12T08:48:28.477643Z | node1 left | -2023-03-12T08:48:28.477680Z | PRIMARY(n=1) | -2023-03-12T08:49:41.706020Z | node1 joined | -2023-03-12T08:49:41.713788Z | PRIMARY(n=2) | -2023-03-12T09:41:30.759927Z | received shutdown | -2023-03-12T09:41:41.775338Z | NON-PRIMARY(n=1) | -2023-03-12T09:41:41.775413Z | SYNCED -> OPEN | -2023-03-12T09:41:41.775442Z | OPEN -> CLOSED | -2023-03-12T09:41:48.745926Z | shutdown complete | - 5.7.40 - (version) -  V  - 8.0.28 -2023-03-12T09:55:30.928545Z | starting(8.0.28) | -2023-03-12T09:59:01.655066Z | started(standalone) | -2023-03-12T10:01:10.488475Z | shutdown complete | -2023-03-12T10:03:03.136053Z | starting(8.0.28) | -2023-03-12T10:03:03.139798Z | started(cluster) | -2023-03-12T10:03:03.157578Z | not safe to bootstrap | -2023-03-12T10:03:03.157601Z | ABORTING | -2023-03-12T10:03:03.157774Z | shutdown complete | -2023-03-12T10:03:03.163682Z | CLOSED -> DESTROYED | -2023-03-12T10:04:12.603100Z | starting(8.0.28) | -2023-03-12T10:04:12.608219Z | started(cluster) | -2023-03-12T10:04:12.609639Z | safe_to_bootstrap: 1 | -2023-03-12T10:04:12.623957Z | bootstrapping | -2023-03-12T10:04:12.628369Z | CLOSED -> OPEN | -2023-03-12T10:04:12.628477Z | PRIMARY(n=1) | -2023-03-12T10:04:12.628792Z | (restored)OPEN -> JOINED | -2023-03-12T10:04:12.628833Z | JOINED -> SYNCED | -2023-03-12T11:23:46.950430Z | received shutdown | -2023-03-12T11:23:56.953018Z | SYNCED -> CLOSED | -2023-03-12T11:24:03.294073Z | shutdown complete | -2023-03-12T11:24:33.315663Z | starting(8.0.28) | -2023-03-12T11:24:33.319800Z | started(cluster) | -2023-03-12T11:24:33.320989Z | safe_to_bootstrap: 1 | -2023-03-12T11:24:33.332251Z | bootstrapping | -2023-03-12T11:24:33.334384Z | CLOSED -> OPEN | -2023-03-12T11:24:33.334467Z | PRIMARY(n=1) | -2023-03-12T11:24:33.334699Z | (restored)OPEN -> JOINED | -2023-03-12T11:24:33.334761Z | JOINED -> SYNCED | -2023-03-12T11:35:14.693312Z | node3 joined | -2023-03-12T11:35:14.695410Z | PRIMARY(n=2) | -2023-03-12T11:35:16.321642Z | SYNCED -> DONOR | -2023-03-12T11:35:16.342707Z | IST to node3(seqno:170403898) | -2023-03-12T11:35:17.118100Z | IST will be used | -2023-03-12T11:35:18.140723Z | finished sending IST to node3 | -2023-03-12T11:35:18.140768Z | DESYNCED -> JOINED | -2023-03-12T11:35:18.141016Z | JOINED -> SYNCED | -2023-03-12T11:35:21.030164Z | node3 left | -2023-03-12T11:35:21.035732Z | node3 left | -2023-03-12T11:35:21.035794Z | PRIMARY(n=1) | -2023-03-12T11:39:20.681083Z | node3 joined | -2023-03-12T11:39:20.683800Z | PRIMARY(n=2) | -2023-03-12T11:39:21.948554Z | SYNCED -> DONOR | -2023-03-12T11:39:21.952242Z | IST to node3(seqno:170403900) | -2023-03-12T11:39:33.420743Z | SST to node3 | -2023-03-12T11:39:38.705565Z | node3 left | -2023-03-12T11:39:38.707686Z | node3 left | -2023-03-12T11:39:38.707695Z | PRIMARY(n=1) | -2023-03-12T11:39:38.734654Z | SST error | -2023-03-12T11:39:38.738833Z | node2 failed to sync ??(node left) | -2023-03-12T11:39:38.738842Z | DESYNCED -> JOINED | -2023-03-12T11:39:38.738942Z | JOINED -> SYNCED | -2023-03-12T12:22:48.704897Z | received shutdown | -2023-03-12T12:22:58.706338Z | SYNCED -> CLOSED | -2023-03-12T12:23:04.677082Z | shutdown complete | -2023-03-12T12:24:36.270274Z | starting(8.0.28) | -2023-03-12T12:24:36.274315Z | started(cluster) | -2023-03-12T12:24:36.275472Z | safe_to_bootstrap: 1 | -2023-03-12T12:24:36.287220Z | bootstrapping | -2023-03-12T12:24:36.290286Z | CLOSED -> OPEN | -2023-03-12T12:24:36.290365Z | PRIMARY(n=1) | -2023-03-12T12:24:36.290625Z | (restored)OPEN -> JOINED | -2023-03-12T12:24:36.290667Z | JOINED -> SYNCED | -2023-03-12T12:29:49.319032Z | node1 joined | -2023-03-12T12:29:49.323505Z | PRIMARY(n=2) | -2023-03-12T12:29:51.443525Z | node1 left | -2023-03-12T12:29:51.445280Z | node1 left | -2023-03-12T12:29:51.445300Z | PRIMARY(n=1) | -2023-03-12T12:48:43.293802Z | |  starting(8.0.28) -2023-03-12T12:48:43.297858Z | |  started(cluster) -2023-03-12T12:48:43.521685Z | node3 joined | -2023-03-12T12:48:43.521846Z | |  node2 joined -2023-03-12T12:48:43.526717Z | PRIMARY(n=2) | -2023-03-12T12:48:43.820825Z | |  CLOSED -> OPEN -2023-03-12T12:48:43.820929Z | |  PRIMARY(n=2) -2023-03-12T12:48:43.822001Z | |  OPEN -> PRIMARY -2023-03-12T12:48:44.597299Z | |  will receive IST(seqno:170403905) -2023-03-12T12:48:44.599341Z | SYNCED -> DONOR |  -2023-03-12T12:48:44.599377Z | |  PRIMARY -> JOINER -2023-03-12T12:48:44.616436Z | IST to node3(seqno:170403905) |  -2023-03-12T12:48:45.044873Z | IST will be used |  -2023-03-12T12:48:46.064764Z | finished sending IST to node3 |  -2023-03-12T12:48:46.064808Z | DESYNCED -> JOINED |  -2023-03-12T12:48:46.065014Z | | got IST from node2 -2023-03-12T12:48:46.065051Z | JOINED -> SYNCED |  -2023-03-12T12:48:54.233973Z | |  wsrep recovery -2023-03-12T12:48:54.269978Z | |  IST received(seqno:170403905) -2023-03-12T12:48:54.272037Z | |  JOINER -> JOINED -2023-03-12T12:48:54.272256Z | |  JOINED -> SYNCED -2023-03-12T13:04:24.476576Z | node3 joined |  -2023-03-12T13:04:24.476642Z | node1 joined |  -2023-03-12T13:04:24.476806Z | |  node1 joined -2023-03-12T13:04:24.476863Z | |  node2 joined -2023-03-12T13:04:24.478964Z | PRIMARY(n=3) |  -2023-03-12T13:04:24.479206Z | |  PRIMARY(n=3) -2023-03-12T13:04:25.732132Z | |  SYNCED -> DONOR -2023-03-12T13:04:25.735999Z | |  IST to node1(seqno:170407335) -2023-03-12T13:04:37.415791Z | |  SST to node1 -2023-03-12T13:04:38.645597Z | node3 joined |  -2023-03-12T13:04:38.645710Z | node1 left |  -2023-03-12T13:04:38.647921Z | |  node2 joined -2023-03-12T13:04:38.647981Z | |  node1 left -2023-03-12T13:04:38.650097Z | |  node1 left -2023-03-12T13:04:38.650125Z | |  PRIMARY(n=2) -2023-03-12T13:04:38.652812Z | node1 left |  -2023-03-12T13:04:38.652875Z | PRIMARY(n=2) |  -2023-03-12T13:04:39.715275Z | |  SST error -2023-03-12T13:04:39.720325Z | node3 failed to sync ??(node left) |  -2023-03-12T13:04:39.720379Z | |  node3 failed to sync ??(node left) -2023-03-12T13:04:39.720388Z | |  DESYNCED -> JOINED -2023-03-12T13:04:39.720600Z | |  JOINED -> SYNCED -2023-03-12T13:12:02.676601Z | received shutdown |  -2023-03-12T13:12:13.679070Z | |  node2 left -2023-03-12T13:12:13.681813Z | |  node2 left -2023-03-12T13:12:13.681867Z | |  PRIMARY(n=1) -2023-03-12T13:12:13.682286Z | NON-PRIMARY(n=1) |  -2023-03-12T13:12:13.682450Z | SYNCED -> OPEN |  -2023-03-12T13:12:13.682565Z | OPEN -> CLOSED |  -2023-03-12T13:12:22.957837Z | shutdown complete |  -2023-03-12T13:13:11.498126Z | starting(8.0.28) |  -2023-03-12T13:13:11.501941Z | started(cluster) |  -2023-03-12T13:13:12.015863Z | node3 joined |  -2023-03-12T13:13:12.015998Z | | node2 joined -2023-03-12T13:13:12.020360Z | | PRIMARY(n=2) -2023-03-12T13:13:12.515546Z | CLOSED -> OPEN |  -2023-03-12T13:13:12.515641Z | PRIMARY(n=2) |  -2023-03-12T13:13:12.516249Z | OPEN -> PRIMARY |  -2023-03-12T13:13:13.245723Z | will receive IST(seqno:170407338) |  -2023-03-12T13:13:13.247750Z | PRIMARY -> JOINER |  -2023-03-12T13:13:13.248065Z | |  SYNCED -> DONOR -2023-03-12T13:13:13.262238Z | |  IST to node2(seqno:170407338) -2023-03-12T13:13:13.863959Z | |  IST will be used -2023-03-12T13:13:14.886853Z | got IST from node3 |  -2023-03-12T13:13:14.886942Z | |  finished sending IST to node2 -2023-03-12T13:13:14.887000Z | |  DESYNCED -> JOINED -2023-03-12T13:13:14.887249Z | |  JOINED -> SYNCED -2023-03-12T13:13:19.031367Z | wsrep recovery |  -2023-03-12T13:13:19.156722Z | IST received(seqno:170407338) |  -2023-03-12T13:13:19.158840Z | JOINER -> JOINED |  -2023-03-12T13:13:19.159057Z | JOINED -> SYNCED |  -2023-03-12T19:35:05.840743Z starting(8.0.28) |  |  -2023-03-12T19:35:05.848542Z started(cluster) |  |  -2023-03-12T19:35:06.375917Z | |  node2 joined -2023-03-12T19:35:06.375974Z | |  node1 joined -2023-03-12T19:35:06.376012Z node3 joined |  |  -2023-03-12T19:35:06.376016Z | node3 joined |  -2023-03-12T19:35:06.376026Z node2 joined |  |  -2023-03-12T19:35:06.376081Z | node1 joined |  -2023-03-12T19:35:06.383186Z | PRIMARY(n=3) |  -2023-03-12T19:35:06.385445Z | |  PRIMARY(n=3) -2023-03-12T19:35:06.875619Z CLOSED -> OPEN |  |  -2023-03-12T19:35:06.875717Z PRIMARY(n=3) |  |  -2023-03-12T19:35:06.876501Z OPEN -> PRIMARY |  |  -2023-03-12T19:35:07.638676Z will receive IST(seqno:178226774) |  |  -2023-03-12T19:35:07.644570Z |  |  SYNCED -> DONOR - -identifier 172.17.0.2 node2 node3 -current path tests/logs/upgrade/node1.log tests/logs/upgrade/node2.log tests/logs/upgrade/node3.log -last known ip 172.17.0.2 172.17.0.3 172.17.0.4 -last known name node2 node3 -mysql version 8.0.28 8.0.28 8.0.28 +identifier 172.17.0.2 node2 node3 +current path tests/logs/upgrade/node1.log tests/logs/upgrade/node2.log tests/logs/upgrade/node3.log +last known ip 172.17.0.2 172.17.0.3 172.17.0.4 +last known name node2 node3 +mysql version 8.0.28 8.0.28 8.0.28 + +2023-03-12T07:24:13.733958Z | starting(5.7.40) | +2023-03-12T07:24:13.771126Z | started(cluster) | +2023-03-12T07:24:14.289375Z | node1 joined | +2023-03-12T07:24:14.289412Z | node3 joined | +2023-03-12T07:24:14.789002Z | CLOSED -> OPEN | +2023-03-12T07:24:14.789075Z | PRIMARY(n=3) | +2023-03-12T07:24:14.789560Z | (restored)OPEN -> JOINED | +2023-03-12T07:24:14.789785Z | JOINED -> SYNCED | +2023-03-12T07:34:47.289292Z | received shutdown | +2023-03-12T07:34:57.286990Z | node1 joined | +2023-03-12T07:34:57.287111Z | node3 left | +2023-03-12T07:34:57.290903Z | node3 left | +2023-03-12T07:35:02.791416Z | (repeated x17)node1 suspected to be down | +2023-03-12T07:35:11.793101Z | node1 suspected to be down | +2023-03-12T07:35:12.293578Z | PRIMARY(n=2) | +2023-03-12T07:35:12.293705Z | NON-PRIMARY(n=1) | +2023-03-12T07:35:12.293723Z | SYNCED -> OPEN | +2023-03-12T07:35:12.293760Z | OPEN -> CLOSED | +2023-03-12T07:35:18.533851Z | shutdown complete | +2023-03-12T07:38:06.673334Z | starting(5.7.40) | +2023-03-12T07:38:06.680025Z | started(cluster) | +2023-03-12T07:38:06.681065Z | safe_to_bootstrap: 1 | +2023-03-12T07:38:06.693619Z | bootstrapping | +2023-03-12T07:38:06.695987Z | CLOSED -> OPEN | +2023-03-12T07:38:06.696042Z | PRIMARY(n=1) | +2023-03-12T07:38:06.696187Z | (restored)OPEN -> JOINED | +2023-03-12T07:38:06.696210Z | JOINED -> SYNCED | +2023-03-12T07:39:27.162350Z | node3 joined | +2023-03-12T07:39:27.164824Z | PRIMARY(n=2) | +2023-03-12T07:43:09.063375Z | node1 joined | +2023-03-12T07:43:09.063430Z | node3 joined | +2023-03-12T07:43:09.065740Z | PRIMARY(n=3) | +2023-03-12T07:49:45.317891Z | received shutdown | +2023-03-12T07:49:55.319157Z | NON-PRIMARY(n=1) | +2023-03-12T07:49:55.319203Z | SYNCED -> OPEN | +2023-03-12T07:49:55.319230Z | OPEN -> CLOSED | +2023-03-12T07:50:00.605309Z | shutdown complete | +2023-03-12T08:46:48.943442Z | starting(5.7.40) | +2023-03-12T08:46:48.947933Z | started(cluster) | +2023-03-12T08:46:48.992365Z | node1 joined | +2023-03-12T08:46:49.463255Z | CLOSED -> OPEN | +2023-03-12T08:46:49.463334Z | PRIMARY(n=2) | +2023-03-12T08:46:49.463988Z | (restored)OPEN -> JOINED | +2023-03-12T08:46:49.464124Z | JOINED -> SYNCED | +2023-03-12T08:48:28.470198Z | node1 left | +2023-03-12T08:48:28.477643Z | node1 left | +2023-03-12T08:48:28.477680Z | PRIMARY(n=1) | +2023-03-12T08:49:41.706020Z | node1 joined | +2023-03-12T08:49:41.713788Z | PRIMARY(n=2) | +2023-03-12T09:41:30.759927Z | received shutdown | +2023-03-12T09:41:41.775338Z | NON-PRIMARY(n=1) | +2023-03-12T09:41:41.775413Z | SYNCED -> OPEN | +2023-03-12T09:41:41.775442Z | OPEN -> CLOSED | +2023-03-12T09:41:48.745926Z | shutdown complete | + 5.7.40 + (version) +  V  + 8.0.28 +2023-03-12T09:55:30.928545Z | starting(8.0.28) | +2023-03-12T09:59:01.655066Z | started(standalone) | +2023-03-12T10:01:10.488475Z | shutdown complete | +2023-03-12T10:03:03.136053Z | starting(8.0.28) | +2023-03-12T10:03:03.139798Z | started(cluster) | +2023-03-12T10:03:03.157578Z | not safe to bootstrap | +2023-03-12T10:03:03.157601Z | ABORTING | +2023-03-12T10:03:03.157774Z | shutdown complete | +2023-03-12T10:03:03.163682Z | CLOSED -> DESTROYED | +2023-03-12T10:04:12.603100Z | starting(8.0.28) | +2023-03-12T10:04:12.608219Z | started(cluster) | +2023-03-12T10:04:12.609639Z | safe_to_bootstrap: 1 | +2023-03-12T10:04:12.623957Z | bootstrapping | +2023-03-12T10:04:12.628369Z | CLOSED -> OPEN | +2023-03-12T10:04:12.628477Z | PRIMARY(n=1) | +2023-03-12T10:04:12.628792Z | (restored)OPEN -> JOINED | +2023-03-12T10:04:12.628833Z | JOINED -> SYNCED | +2023-03-12T11:23:46.950430Z | received shutdown | +2023-03-12T11:23:56.953018Z | SYNCED -> CLOSED | +2023-03-12T11:24:03.294073Z | shutdown complete | +2023-03-12T11:24:33.315663Z | starting(8.0.28) | +2023-03-12T11:24:33.319800Z | started(cluster) | +2023-03-12T11:24:33.320989Z | safe_to_bootstrap: 1 | +2023-03-12T11:24:33.332251Z | bootstrapping | +2023-03-12T11:24:33.334384Z | CLOSED -> OPEN | +2023-03-12T11:24:33.334467Z | PRIMARY(n=1) | +2023-03-12T11:24:33.334699Z | (restored)OPEN -> JOINED | +2023-03-12T11:24:33.334761Z | JOINED -> SYNCED | +2023-03-12T11:35:14.693312Z | node3 joined | +2023-03-12T11:35:14.695410Z | PRIMARY(n=2) | +2023-03-12T11:35:16.321586Z | local node will resync node3 | +2023-03-12T11:35:16.321642Z | SYNCED -> DONOR | +2023-03-12T11:35:16.342707Z | IST to node3(seqno:170403898) | +2023-03-12T11:35:17.118100Z | IST will be used | +2023-03-12T11:35:18.140723Z | finished sending IST to node3 | +2023-03-12T11:35:18.140768Z | DESYNCED -> JOINED | +2023-03-12T11:35:18.141016Z | JOINED -> SYNCED | +2023-03-12T11:35:21.030164Z | node3 left | +2023-03-12T11:35:21.035732Z | node3 left | +2023-03-12T11:35:21.035794Z | PRIMARY(n=1) | +2023-03-12T11:39:20.681083Z | node3 joined | +2023-03-12T11:39:20.683800Z | PRIMARY(n=2) | +2023-03-12T11:39:21.948501Z | local node will resync node3 | +2023-03-12T11:39:21.948554Z | SYNCED -> DONOR | +2023-03-12T11:39:21.952242Z | IST to node3(seqno:170403900) | +2023-03-12T11:39:33.420743Z | SST to node3 | +2023-03-12T11:39:38.705565Z | node3 left | +2023-03-12T11:39:38.707686Z | node3 left | +2023-03-12T11:39:38.707695Z | PRIMARY(n=1) | +2023-03-12T11:39:38.734654Z | SST error | +2023-03-12T11:39:38.738833Z | node2 failed to sync ??(node left) | +2023-03-12T11:39:38.738842Z | DESYNCED -> JOINED | +2023-03-12T11:39:38.738942Z | JOINED -> SYNCED | +2023-03-12T12:22:48.704897Z | received shutdown | +2023-03-12T12:22:58.706338Z | SYNCED -> CLOSED | +2023-03-12T12:23:04.677082Z | shutdown complete | +2023-03-12T12:24:36.270274Z | starting(8.0.28) | +2023-03-12T12:24:36.274315Z | started(cluster) | +2023-03-12T12:24:36.275472Z | safe_to_bootstrap: 1 | +2023-03-12T12:24:36.287220Z | bootstrapping | +2023-03-12T12:24:36.290286Z | CLOSED -> OPEN | +2023-03-12T12:24:36.290365Z | PRIMARY(n=1) | +2023-03-12T12:24:36.290625Z | (restored)OPEN -> JOINED | +2023-03-12T12:24:36.290667Z | JOINED -> SYNCED | +2023-03-12T12:29:49.319032Z | node1 joined | +2023-03-12T12:29:49.323505Z | PRIMARY(n=2) | +2023-03-12T12:29:51.443525Z | node1 left | +2023-03-12T12:29:51.445280Z | node1 left | +2023-03-12T12:29:51.445300Z | PRIMARY(n=1) | +2023-03-12T12:48:43.293802Z | |  starting(8.0.28) +2023-03-12T12:48:43.297858Z | |  started(cluster) +2023-03-12T12:48:43.521685Z | node3 joined | +2023-03-12T12:48:43.521846Z | |  node2 joined +2023-03-12T12:48:43.526717Z | PRIMARY(n=2) | +2023-03-12T12:48:43.820825Z | |  CLOSED -> OPEN +2023-03-12T12:48:43.820929Z | |  PRIMARY(n=2) +2023-03-12T12:48:43.822001Z | |  OPEN -> PRIMARY +2023-03-12T12:48:44.597299Z | |  will receive IST(seqno:170403905) +2023-03-12T12:48:44.599287Z | local node will resync node3 |  +2023-03-12T12:48:44.599341Z | SYNCED -> DONOR |  +2023-03-12T12:48:44.599346Z | |  node2 will resync local node +2023-03-12T12:48:44.599377Z | |  PRIMARY -> JOINER +2023-03-12T12:48:44.616436Z | IST to node3(seqno:170403905) |  +2023-03-12T12:48:45.044873Z | IST will be used |  +2023-03-12T12:48:46.064764Z | finished sending IST to node3 |  +2023-03-12T12:48:46.064808Z | DESYNCED -> JOINED |  +2023-03-12T12:48:46.065014Z | | got IST from node2 +2023-03-12T12:48:46.065051Z | JOINED -> SYNCED |  +2023-03-12T12:48:54.233973Z | |  wsrep recovery +2023-03-12T12:48:54.269978Z | |  IST received(seqno:170403905) +2023-03-12T12:48:54.272037Z | |  JOINER -> JOINED +2023-03-12T12:48:54.272256Z | |  JOINED -> SYNCED +2023-03-12T13:04:24.476576Z | node3 joined |  +2023-03-12T13:04:24.476642Z | node1 joined |  +2023-03-12T13:04:24.476806Z | |  node1 joined +2023-03-12T13:04:24.476863Z | |  node2 joined +2023-03-12T13:04:24.478964Z | PRIMARY(n=3) |  +2023-03-12T13:04:24.479206Z | |  PRIMARY(n=3) +2023-03-12T13:04:25.731994Z | node3 will resync node1 |  +2023-03-12T13:04:25.732124Z | |  local node will resync node1 +2023-03-12T13:04:25.732132Z | |  SYNCED -> DONOR +2023-03-12T13:04:25.735999Z | |  IST to node1(seqno:170407335) +2023-03-12T13:04:37.415791Z | |  SST to node1 +2023-03-12T13:04:38.645597Z | node3 joined |  +2023-03-12T13:04:38.645710Z | node1 left |  +2023-03-12T13:04:38.647921Z | |  node2 joined +2023-03-12T13:04:38.647981Z | |  node1 left +2023-03-12T13:04:38.650097Z | |  node1 left +2023-03-12T13:04:38.650125Z | |  PRIMARY(n=2) +2023-03-12T13:04:38.652812Z | node1 left |  +2023-03-12T13:04:38.652875Z | PRIMARY(n=2) |  +2023-03-12T13:04:39.715275Z | |  SST error +2023-03-12T13:04:39.720325Z | node3 failed to sync ??(node left) |  +2023-03-12T13:04:39.720379Z | |  node3 failed to sync ??(node left) +2023-03-12T13:04:39.720388Z | |  DESYNCED -> JOINED +2023-03-12T13:04:39.720600Z | |  JOINED -> SYNCED +2023-03-12T13:12:02.676601Z | received shutdown |  +2023-03-12T13:12:13.679070Z | |  node2 left +2023-03-12T13:12:13.681813Z | |  node2 left +2023-03-12T13:12:13.681867Z | |  PRIMARY(n=1) +2023-03-12T13:12:13.682286Z | NON-PRIMARY(n=1) |  +2023-03-12T13:12:13.682450Z | SYNCED -> OPEN |  +2023-03-12T13:12:13.682565Z | OPEN -> CLOSED |  +2023-03-12T13:12:22.957837Z | shutdown complete |  +2023-03-12T13:13:11.498126Z | starting(8.0.28) |  +2023-03-12T13:13:11.501941Z | started(cluster) |  +2023-03-12T13:13:12.015863Z | node3 joined |  +2023-03-12T13:13:12.015998Z | | node2 joined +2023-03-12T13:13:12.020360Z | | PRIMARY(n=2) +2023-03-12T13:13:12.515546Z | CLOSED -> OPEN |  +2023-03-12T13:13:12.515641Z | PRIMARY(n=2) |  +2023-03-12T13:13:12.516249Z | OPEN -> PRIMARY |  +2023-03-12T13:13:13.245723Z | will receive IST(seqno:170407338) |  +2023-03-12T13:13:13.247714Z | node3 will resync local node |  +2023-03-12T13:13:13.247750Z | PRIMARY -> JOINER |  +2023-03-12T13:13:13.248015Z | |  local node will resync node2 +2023-03-12T13:13:13.248065Z | |  SYNCED -> DONOR +2023-03-12T13:13:13.262238Z | |  IST to node2(seqno:170407338) +2023-03-12T13:13:13.863959Z | |  IST will be used +2023-03-12T13:13:14.886853Z | got IST from node3 |  +2023-03-12T13:13:14.886942Z | |  finished sending IST to node2 +2023-03-12T13:13:14.887000Z | |  DESYNCED -> JOINED +2023-03-12T13:13:14.887249Z | |  JOINED -> SYNCED +2023-03-12T13:13:19.031367Z | wsrep recovery |  +2023-03-12T13:13:19.156722Z | IST received(seqno:170407338) |  +2023-03-12T13:13:19.158840Z | JOINER -> JOINED |  +2023-03-12T13:13:19.159057Z | JOINED -> SYNCED |  +2023-03-12T19:35:05.840743Z starting(8.0.28) |  |  +2023-03-12T19:35:05.848542Z started(cluster) |  |  +2023-03-12T19:35:06.375917Z | |  node2 joined +2023-03-12T19:35:06.375974Z | |  node1 joined +2023-03-12T19:35:06.376012Z node3 joined |  |  +2023-03-12T19:35:06.376016Z | node3 joined |  +2023-03-12T19:35:06.376026Z node2 joined |  |  +2023-03-12T19:35:06.376081Z | node1 joined |  +2023-03-12T19:35:06.383186Z | PRIMARY(n=3) |  +2023-03-12T19:35:06.385445Z | |  PRIMARY(n=3) +2023-03-12T19:35:06.875619Z CLOSED -> OPEN |  |  +2023-03-12T19:35:06.875717Z PRIMARY(n=3) |  |  +2023-03-12T19:35:06.876501Z OPEN -> PRIMARY |  |  +2023-03-12T19:35:07.638676Z will receive IST(seqno:178226774) |  |  +2023-03-12T19:35:07.644560Z |  |  local node will resync node1 +2023-03-12T19:35:07.644570Z |  |  SYNCED -> DONOR + +identifier 172.17.0.2 node2 node3 +current path tests/logs/upgrade/node1.log tests/logs/upgrade/node2.log tests/logs/upgrade/node3.log +last known ip 172.17.0.2 172.17.0.3 172.17.0.4 +last known name node2 node3 +mysql version 8.0.28 8.0.28 8.0.28 diff --git a/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_until_hiding_1_node b/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_until_hiding_1_node index 8481e2bc..ac937c3e 100644 --- a/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_until_hiding_1_node +++ b/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_until_hiding_1_node @@ -1,204 +1,214 @@ -identifier node2 node3 -current path tests/logs/upgrade/node2.log tests/logs/upgrade/node3.log -last known ip 172.17.0.3 172.17.0.4 -last known name node2 node3 -mysql version 8.0.28 8.0.28 - -2023-03-12T07:24:13.733958Z starting(5.7.40) | -2023-03-12T07:24:13.771126Z started(cluster) | -2023-03-12T07:24:14.289375Z node1 joined | -2023-03-12T07:24:14.289412Z node3 joined | -2023-03-12T07:24:14.789002Z CLOSED -> OPEN | -2023-03-12T07:24:14.789075Z PRIMARY(n=3) | -2023-03-12T07:24:14.789560Z (restored)OPEN -> JOINED | -2023-03-12T07:24:14.789785Z JOINED -> SYNCED | -2023-03-12T07:34:47.289292Z received shutdown | -2023-03-12T07:34:57.286990Z node1 joined | -2023-03-12T07:34:57.287111Z node3 left | -2023-03-12T07:34:57.290903Z node3 left | -2023-03-12T07:35:12.293578Z PRIMARY(n=2) | -2023-03-12T07:35:12.293705Z NON-PRIMARY(n=1) | -2023-03-12T07:35:12.293723Z SYNCED -> OPEN | -2023-03-12T07:35:12.293760Z OPEN -> CLOSED | -2023-03-12T07:35:18.533851Z shutdown complete | -2023-03-12T07:38:06.673334Z starting(5.7.40) | -2023-03-12T07:38:06.680025Z started(cluster) | -2023-03-12T07:38:06.681065Z safe_to_bootstrap: 1 | -2023-03-12T07:38:06.693619Z bootstrapping | -2023-03-12T07:38:06.695987Z CLOSED -> OPEN | -2023-03-12T07:38:06.696042Z PRIMARY(n=1) | -2023-03-12T07:38:06.696187Z (restored)OPEN -> JOINED | -2023-03-12T07:38:06.696210Z JOINED -> SYNCED | -2023-03-12T07:39:27.162350Z node3 joined | -2023-03-12T07:39:27.164824Z PRIMARY(n=2) | -2023-03-12T07:43:09.063375Z node1 joined | -2023-03-12T07:43:09.063430Z node3 joined | -2023-03-12T07:43:09.065740Z PRIMARY(n=3) | -2023-03-12T07:49:45.317891Z received shutdown | -2023-03-12T07:49:55.319157Z NON-PRIMARY(n=1) | -2023-03-12T07:49:55.319203Z SYNCED -> OPEN | -2023-03-12T07:49:55.319230Z OPEN -> CLOSED | -2023-03-12T07:50:00.605309Z shutdown complete | -2023-03-12T08:46:48.943442Z starting(5.7.40) | -2023-03-12T08:46:48.947933Z started(cluster) | -2023-03-12T08:46:48.992365Z node1 joined | -2023-03-12T08:46:49.463255Z CLOSED -> OPEN | -2023-03-12T08:46:49.463334Z PRIMARY(n=2) | -2023-03-12T08:46:49.463988Z (restored)OPEN -> JOINED | -2023-03-12T08:46:49.464124Z JOINED -> SYNCED | -2023-03-12T08:48:28.470198Z node1 left | -2023-03-12T08:48:28.477643Z node1 left | -2023-03-12T08:48:28.477680Z PRIMARY(n=1) | -2023-03-12T08:49:41.706020Z node1 joined | -2023-03-12T08:49:41.713788Z PRIMARY(n=2) | -2023-03-12T09:41:30.759927Z received shutdown | -2023-03-12T09:41:41.775338Z NON-PRIMARY(n=1) | -2023-03-12T09:41:41.775413Z SYNCED -> OPEN | -2023-03-12T09:41:41.775442Z OPEN -> CLOSED | -2023-03-12T09:41:48.745926Z shutdown complete | - 5.7.40 - (version) -  V  - 8.0.28 -2023-03-12T09:55:30.928545Z starting(8.0.28) | -2023-03-12T09:59:01.655066Z started(standalone) | -2023-03-12T10:01:10.488475Z shutdown complete | -2023-03-12T10:03:03.136053Z starting(8.0.28) | -2023-03-12T10:03:03.139798Z started(cluster) | -2023-03-12T10:03:03.157578Z not safe to bootstrap | -2023-03-12T10:03:03.157601Z ABORTING | -2023-03-12T10:03:03.157774Z shutdown complete | -2023-03-12T10:03:03.163682Z CLOSED -> DESTROYED | -2023-03-12T10:04:12.603100Z starting(8.0.28) | -2023-03-12T10:04:12.608219Z started(cluster) | -2023-03-12T10:04:12.609639Z safe_to_bootstrap: 1 | -2023-03-12T10:04:12.623957Z bootstrapping | -2023-03-12T10:04:12.628369Z CLOSED -> OPEN | -2023-03-12T10:04:12.628477Z PRIMARY(n=1) | -2023-03-12T10:04:12.628792Z (restored)OPEN -> JOINED | -2023-03-12T10:04:12.628833Z JOINED -> SYNCED | -2023-03-12T11:23:46.950430Z received shutdown | -2023-03-12T11:23:56.953018Z SYNCED -> CLOSED | -2023-03-12T11:24:03.294073Z shutdown complete | -2023-03-12T11:24:33.315663Z starting(8.0.28) | -2023-03-12T11:24:33.319800Z started(cluster) | -2023-03-12T11:24:33.320989Z safe_to_bootstrap: 1 | -2023-03-12T11:24:33.332251Z bootstrapping | -2023-03-12T11:24:33.334384Z CLOSED -> OPEN | -2023-03-12T11:24:33.334467Z PRIMARY(n=1) | -2023-03-12T11:24:33.334699Z (restored)OPEN -> JOINED | -2023-03-12T11:24:33.334761Z JOINED -> SYNCED | -2023-03-12T11:35:14.693312Z node3 joined | -2023-03-12T11:35:14.695410Z PRIMARY(n=2) | -2023-03-12T11:35:16.321642Z SYNCED -> DONOR | -2023-03-12T11:35:16.342707Z IST to node3(seqno:170403898) | -2023-03-12T11:35:17.118100Z IST will be used | -2023-03-12T11:35:18.140723Z finished sending IST to node3 | -2023-03-12T11:35:18.140768Z DESYNCED -> JOINED | -2023-03-12T11:35:18.141016Z JOINED -> SYNCED | -2023-03-12T11:35:21.030164Z node3 left | -2023-03-12T11:35:21.035732Z node3 left | -2023-03-12T11:35:21.035794Z PRIMARY(n=1) | -2023-03-12T11:39:20.681083Z node3 joined | -2023-03-12T11:39:20.683800Z PRIMARY(n=2) | -2023-03-12T11:39:21.948554Z SYNCED -> DONOR | -2023-03-12T11:39:21.952242Z IST to node3(seqno:170403900) | -2023-03-12T11:39:33.420743Z SST to node3 | -2023-03-12T11:39:38.705565Z node3 left | -2023-03-12T11:39:38.707686Z node3 left | -2023-03-12T11:39:38.707695Z PRIMARY(n=1) | -2023-03-12T11:39:38.734654Z SST error | -2023-03-12T11:39:38.738833Z node2 failed to sync ??(node left) | -2023-03-12T11:39:38.738842Z DESYNCED -> JOINED | -2023-03-12T11:39:38.738942Z JOINED -> SYNCED | -2023-03-12T12:22:48.704897Z received shutdown | -2023-03-12T12:22:58.706338Z SYNCED -> CLOSED | -2023-03-12T12:23:04.677082Z shutdown complete | -2023-03-12T12:24:36.270274Z starting(8.0.28) | -2023-03-12T12:24:36.274315Z started(cluster) | -2023-03-12T12:24:36.275472Z safe_to_bootstrap: 1 | -2023-03-12T12:24:36.287220Z bootstrapping | -2023-03-12T12:24:36.290286Z CLOSED -> OPEN | -2023-03-12T12:24:36.290365Z PRIMARY(n=1) | -2023-03-12T12:24:36.290625Z (restored)OPEN -> JOINED | -2023-03-12T12:24:36.290667Z JOINED -> SYNCED | -2023-03-12T12:29:49.319032Z node1 joined | -2023-03-12T12:29:49.323505Z PRIMARY(n=2) | -2023-03-12T12:29:51.443525Z node1 left | -2023-03-12T12:29:51.445280Z node1 left | -2023-03-12T12:29:51.445300Z PRIMARY(n=1) | -2023-03-12T12:48:43.293802Z |  starting(8.0.28) -2023-03-12T12:48:43.297858Z |  started(cluster) -2023-03-12T12:48:43.521685Z node3 joined | -2023-03-12T12:48:43.521846Z |  node2 joined -2023-03-12T12:48:43.526717Z PRIMARY(n=2) | -2023-03-12T12:48:43.820825Z |  CLOSED -> OPEN -2023-03-12T12:48:43.820929Z |  PRIMARY(n=2) -2023-03-12T12:48:43.822001Z |  OPEN -> PRIMARY -2023-03-12T12:48:44.597299Z |  will receive IST(seqno:170403905) -2023-03-12T12:48:44.599341Z SYNCED -> DONOR |  -2023-03-12T12:48:44.599377Z |  PRIMARY -> JOINER -2023-03-12T12:48:44.616436Z IST to node3(seqno:170403905) |  -2023-03-12T12:48:45.044873Z IST will be used |  -2023-03-12T12:48:46.064764Z finished sending IST to node3 |  -2023-03-12T12:48:46.064808Z DESYNCED -> JOINED |  -2023-03-12T12:48:46.065014Z | got IST from node2 -2023-03-12T12:48:46.065051Z JOINED -> SYNCED |  -2023-03-12T12:48:54.233973Z |  wsrep recovery -2023-03-12T12:48:54.269978Z |  IST received(seqno:170403905) -2023-03-12T12:48:54.272037Z |  JOINER -> JOINED -2023-03-12T12:48:54.272256Z |  JOINED -> SYNCED -2023-03-12T13:04:24.476576Z node3 joined |  -2023-03-12T13:04:24.476642Z node1 joined |  -2023-03-12T13:04:24.476806Z |  node1 joined -2023-03-12T13:04:24.476863Z |  node2 joined -2023-03-12T13:04:24.478964Z PRIMARY(n=3) |  -2023-03-12T13:04:24.479206Z |  PRIMARY(n=3) -2023-03-12T13:04:25.732132Z |  SYNCED -> DONOR -2023-03-12T13:04:25.735999Z |  IST to node1(seqno:170407335) -2023-03-12T13:04:37.415791Z |  SST to node1 -2023-03-12T13:04:38.645597Z node3 joined |  -2023-03-12T13:04:38.645710Z node1 left |  -2023-03-12T13:04:38.647921Z |  node2 joined -2023-03-12T13:04:38.647981Z |  node1 left -2023-03-12T13:04:38.650097Z |  node1 left -2023-03-12T13:04:38.650125Z |  PRIMARY(n=2) -2023-03-12T13:04:38.652812Z node1 left |  -2023-03-12T13:04:38.652875Z PRIMARY(n=2) |  -2023-03-12T13:04:39.715275Z |  SST error -2023-03-12T13:04:39.720325Z node3 failed to sync ??(node left) |  -2023-03-12T13:04:39.720379Z |  node3 failed to sync ??(node left) -2023-03-12T13:04:39.720388Z |  DESYNCED -> JOINED -2023-03-12T13:04:39.720600Z |  JOINED -> SYNCED -2023-03-12T13:12:02.676601Z received shutdown |  -2023-03-12T13:12:13.679070Z |  node2 left -2023-03-12T13:12:13.681813Z |  node2 left -2023-03-12T13:12:13.681867Z |  PRIMARY(n=1) -2023-03-12T13:12:13.682286Z NON-PRIMARY(n=1) |  -2023-03-12T13:12:13.682450Z SYNCED -> OPEN |  -2023-03-12T13:12:13.682565Z OPEN -> CLOSED |  -2023-03-12T13:12:22.957837Z shutdown complete |  -2023-03-12T13:13:11.498126Z starting(8.0.28) |  -2023-03-12T13:13:11.501941Z started(cluster) |  -2023-03-12T13:13:12.015863Z node3 joined |  -2023-03-12T13:13:12.015998Z | node2 joined -2023-03-12T13:13:12.020360Z | PRIMARY(n=2) -2023-03-12T13:13:12.515546Z CLOSED -> OPEN |  -2023-03-12T13:13:12.515641Z PRIMARY(n=2) |  -2023-03-12T13:13:12.516249Z OPEN -> PRIMARY |  -2023-03-12T13:13:13.245723Z will receive IST(seqno:170407338) |  -2023-03-12T13:13:13.247750Z PRIMARY -> JOINER |  -2023-03-12T13:13:13.248065Z |  SYNCED -> DONOR -2023-03-12T13:13:13.262238Z |  IST to node2(seqno:170407338) -2023-03-12T13:13:13.863959Z |  IST will be used -2023-03-12T13:13:14.886853Z got IST from node3 |  -2023-03-12T13:13:14.886942Z |  finished sending IST to node2 -2023-03-12T13:13:14.887000Z |  DESYNCED -> JOINED -2023-03-12T13:13:14.887249Z |  JOINED -> SYNCED -2023-03-12T13:13:19.031367Z wsrep recovery |  - -identifier node2 node3 -current path tests/logs/upgrade/node2.log tests/logs/upgrade/node3.log -last known ip 172.17.0.3 172.17.0.4 -last known name node2 node3 -mysql version 8.0.28 8.0.28 +identifier node2 node3 +current path tests/logs/upgrade/node2.log tests/logs/upgrade/node3.log +last known ip 172.17.0.3 172.17.0.4 +last known name node2 node3 +mysql version 8.0.28 8.0.28 + +2023-03-12T07:24:13.733958Z starting(5.7.40) | +2023-03-12T07:24:13.771126Z started(cluster) | +2023-03-12T07:24:14.289375Z node1 joined | +2023-03-12T07:24:14.289412Z node3 joined | +2023-03-12T07:24:14.789002Z CLOSED -> OPEN | +2023-03-12T07:24:14.789075Z PRIMARY(n=3) | +2023-03-12T07:24:14.789560Z (restored)OPEN -> JOINED | +2023-03-12T07:24:14.789785Z JOINED -> SYNCED | +2023-03-12T07:34:47.289292Z received shutdown | +2023-03-12T07:34:57.286990Z node1 joined | +2023-03-12T07:34:57.287111Z node3 left | +2023-03-12T07:34:57.290903Z node3 left | +2023-03-12T07:35:02.791416Z (repeated x17)node1 suspected to be down | +2023-03-12T07:35:11.793101Z node1 suspected to be down | +2023-03-12T07:35:12.293578Z PRIMARY(n=2) | +2023-03-12T07:35:12.293705Z NON-PRIMARY(n=1) | +2023-03-12T07:35:12.293723Z SYNCED -> OPEN | +2023-03-12T07:35:12.293760Z OPEN -> CLOSED | +2023-03-12T07:35:18.533851Z shutdown complete | +2023-03-12T07:38:06.673334Z starting(5.7.40) | +2023-03-12T07:38:06.680025Z started(cluster) | +2023-03-12T07:38:06.681065Z safe_to_bootstrap: 1 | +2023-03-12T07:38:06.693619Z bootstrapping | +2023-03-12T07:38:06.695987Z CLOSED -> OPEN | +2023-03-12T07:38:06.696042Z PRIMARY(n=1) | +2023-03-12T07:38:06.696187Z (restored)OPEN -> JOINED | +2023-03-12T07:38:06.696210Z JOINED -> SYNCED | +2023-03-12T07:39:27.162350Z node3 joined | +2023-03-12T07:39:27.164824Z PRIMARY(n=2) | +2023-03-12T07:43:09.063375Z node1 joined | +2023-03-12T07:43:09.063430Z node3 joined | +2023-03-12T07:43:09.065740Z PRIMARY(n=3) | +2023-03-12T07:49:45.317891Z received shutdown | +2023-03-12T07:49:55.319157Z NON-PRIMARY(n=1) | +2023-03-12T07:49:55.319203Z SYNCED -> OPEN | +2023-03-12T07:49:55.319230Z OPEN -> CLOSED | +2023-03-12T07:50:00.605309Z shutdown complete | +2023-03-12T08:46:48.943442Z starting(5.7.40) | +2023-03-12T08:46:48.947933Z started(cluster) | +2023-03-12T08:46:48.992365Z node1 joined | +2023-03-12T08:46:49.463255Z CLOSED -> OPEN | +2023-03-12T08:46:49.463334Z PRIMARY(n=2) | +2023-03-12T08:46:49.463988Z (restored)OPEN -> JOINED | +2023-03-12T08:46:49.464124Z JOINED -> SYNCED | +2023-03-12T08:48:28.470198Z node1 left | +2023-03-12T08:48:28.477643Z node1 left | +2023-03-12T08:48:28.477680Z PRIMARY(n=1) | +2023-03-12T08:49:41.706020Z node1 joined | +2023-03-12T08:49:41.713788Z PRIMARY(n=2) | +2023-03-12T09:41:30.759927Z received shutdown | +2023-03-12T09:41:41.775338Z NON-PRIMARY(n=1) | +2023-03-12T09:41:41.775413Z SYNCED -> OPEN | +2023-03-12T09:41:41.775442Z OPEN -> CLOSED | +2023-03-12T09:41:48.745926Z shutdown complete | + 5.7.40 + (version) +  V  + 8.0.28 +2023-03-12T09:55:30.928545Z starting(8.0.28) | +2023-03-12T09:59:01.655066Z started(standalone) | +2023-03-12T10:01:10.488475Z shutdown complete | +2023-03-12T10:03:03.136053Z starting(8.0.28) | +2023-03-12T10:03:03.139798Z started(cluster) | +2023-03-12T10:03:03.157578Z not safe to bootstrap | +2023-03-12T10:03:03.157601Z ABORTING | +2023-03-12T10:03:03.157774Z shutdown complete | +2023-03-12T10:03:03.163682Z CLOSED -> DESTROYED | +2023-03-12T10:04:12.603100Z starting(8.0.28) | +2023-03-12T10:04:12.608219Z started(cluster) | +2023-03-12T10:04:12.609639Z safe_to_bootstrap: 1 | +2023-03-12T10:04:12.623957Z bootstrapping | +2023-03-12T10:04:12.628369Z CLOSED -> OPEN | +2023-03-12T10:04:12.628477Z PRIMARY(n=1) | +2023-03-12T10:04:12.628792Z (restored)OPEN -> JOINED | +2023-03-12T10:04:12.628833Z JOINED -> SYNCED | +2023-03-12T11:23:46.950430Z received shutdown | +2023-03-12T11:23:56.953018Z SYNCED -> CLOSED | +2023-03-12T11:24:03.294073Z shutdown complete | +2023-03-12T11:24:33.315663Z starting(8.0.28) | +2023-03-12T11:24:33.319800Z started(cluster) | +2023-03-12T11:24:33.320989Z safe_to_bootstrap: 1 | +2023-03-12T11:24:33.332251Z bootstrapping | +2023-03-12T11:24:33.334384Z CLOSED -> OPEN | +2023-03-12T11:24:33.334467Z PRIMARY(n=1) | +2023-03-12T11:24:33.334699Z (restored)OPEN -> JOINED | +2023-03-12T11:24:33.334761Z JOINED -> SYNCED | +2023-03-12T11:35:14.693312Z node3 joined | +2023-03-12T11:35:14.695410Z PRIMARY(n=2) | +2023-03-12T11:35:16.321586Z local node will resync node3 | +2023-03-12T11:35:16.321642Z SYNCED -> DONOR | +2023-03-12T11:35:16.342707Z IST to node3(seqno:170403898) | +2023-03-12T11:35:17.118100Z IST will be used | +2023-03-12T11:35:18.140723Z finished sending IST to node3 | +2023-03-12T11:35:18.140768Z DESYNCED -> JOINED | +2023-03-12T11:35:18.141016Z JOINED -> SYNCED | +2023-03-12T11:35:21.030164Z node3 left | +2023-03-12T11:35:21.035732Z node3 left | +2023-03-12T11:35:21.035794Z PRIMARY(n=1) | +2023-03-12T11:39:20.681083Z node3 joined | +2023-03-12T11:39:20.683800Z PRIMARY(n=2) | +2023-03-12T11:39:21.948501Z local node will resync node3 | +2023-03-12T11:39:21.948554Z SYNCED -> DONOR | +2023-03-12T11:39:21.952242Z IST to node3(seqno:170403900) | +2023-03-12T11:39:33.420743Z SST to node3 | +2023-03-12T11:39:38.705565Z node3 left | +2023-03-12T11:39:38.707686Z node3 left | +2023-03-12T11:39:38.707695Z PRIMARY(n=1) | +2023-03-12T11:39:38.734654Z SST error | +2023-03-12T11:39:38.738833Z node2 failed to sync ??(node left) | +2023-03-12T11:39:38.738842Z DESYNCED -> JOINED | +2023-03-12T11:39:38.738942Z JOINED -> SYNCED | +2023-03-12T12:22:48.704897Z received shutdown | +2023-03-12T12:22:58.706338Z SYNCED -> CLOSED | +2023-03-12T12:23:04.677082Z shutdown complete | +2023-03-12T12:24:36.270274Z starting(8.0.28) | +2023-03-12T12:24:36.274315Z started(cluster) | +2023-03-12T12:24:36.275472Z safe_to_bootstrap: 1 | +2023-03-12T12:24:36.287220Z bootstrapping | +2023-03-12T12:24:36.290286Z CLOSED -> OPEN | +2023-03-12T12:24:36.290365Z PRIMARY(n=1) | +2023-03-12T12:24:36.290625Z (restored)OPEN -> JOINED | +2023-03-12T12:24:36.290667Z JOINED -> SYNCED | +2023-03-12T12:29:49.319032Z node1 joined | +2023-03-12T12:29:49.323505Z PRIMARY(n=2) | +2023-03-12T12:29:51.443525Z node1 left | +2023-03-12T12:29:51.445280Z node1 left | +2023-03-12T12:29:51.445300Z PRIMARY(n=1) | +2023-03-12T12:48:43.293802Z |  starting(8.0.28) +2023-03-12T12:48:43.297858Z |  started(cluster) +2023-03-12T12:48:43.521685Z node3 joined | +2023-03-12T12:48:43.521846Z |  node2 joined +2023-03-12T12:48:43.526717Z PRIMARY(n=2) | +2023-03-12T12:48:43.820825Z |  CLOSED -> OPEN +2023-03-12T12:48:43.820929Z |  PRIMARY(n=2) +2023-03-12T12:48:43.822001Z |  OPEN -> PRIMARY +2023-03-12T12:48:44.597299Z |  will receive IST(seqno:170403905) +2023-03-12T12:48:44.599287Z local node will resync node3 |  +2023-03-12T12:48:44.599341Z SYNCED -> DONOR |  +2023-03-12T12:48:44.599346Z |  node2 will resync local node +2023-03-12T12:48:44.599377Z |  PRIMARY -> JOINER +2023-03-12T12:48:44.616436Z IST to node3(seqno:170403905) |  +2023-03-12T12:48:45.044873Z IST will be used |  +2023-03-12T12:48:46.064764Z finished sending IST to node3 |  +2023-03-12T12:48:46.064808Z DESYNCED -> JOINED |  +2023-03-12T12:48:46.065014Z | got IST from node2 +2023-03-12T12:48:46.065051Z JOINED -> SYNCED |  +2023-03-12T12:48:54.233973Z |  wsrep recovery +2023-03-12T12:48:54.269978Z |  IST received(seqno:170403905) +2023-03-12T12:48:54.272037Z |  JOINER -> JOINED +2023-03-12T12:48:54.272256Z |  JOINED -> SYNCED +2023-03-12T13:04:24.476576Z node3 joined |  +2023-03-12T13:04:24.476642Z node1 joined |  +2023-03-12T13:04:24.476806Z |  node1 joined +2023-03-12T13:04:24.476863Z |  node2 joined +2023-03-12T13:04:24.478964Z PRIMARY(n=3) |  +2023-03-12T13:04:24.479206Z |  PRIMARY(n=3) +2023-03-12T13:04:25.731994Z node3 will resync node1 |  +2023-03-12T13:04:25.732124Z |  local node will resync node1 +2023-03-12T13:04:25.732132Z |  SYNCED -> DONOR +2023-03-12T13:04:25.735999Z |  IST to node1(seqno:170407335) +2023-03-12T13:04:37.415791Z |  SST to node1 +2023-03-12T13:04:38.645597Z node3 joined |  +2023-03-12T13:04:38.645710Z node1 left |  +2023-03-12T13:04:38.647921Z |  node2 joined +2023-03-12T13:04:38.647981Z |  node1 left +2023-03-12T13:04:38.650097Z |  node1 left +2023-03-12T13:04:38.650125Z |  PRIMARY(n=2) +2023-03-12T13:04:38.652812Z node1 left |  +2023-03-12T13:04:38.652875Z PRIMARY(n=2) |  +2023-03-12T13:04:39.715275Z |  SST error +2023-03-12T13:04:39.720325Z node3 failed to sync ??(node left) |  +2023-03-12T13:04:39.720379Z |  node3 failed to sync ??(node left) +2023-03-12T13:04:39.720388Z |  DESYNCED -> JOINED +2023-03-12T13:04:39.720600Z |  JOINED -> SYNCED +2023-03-12T13:12:02.676601Z received shutdown |  +2023-03-12T13:12:13.679070Z |  node2 left +2023-03-12T13:12:13.681813Z |  node2 left +2023-03-12T13:12:13.681867Z |  PRIMARY(n=1) +2023-03-12T13:12:13.682286Z NON-PRIMARY(n=1) |  +2023-03-12T13:12:13.682450Z SYNCED -> OPEN |  +2023-03-12T13:12:13.682565Z OPEN -> CLOSED |  +2023-03-12T13:12:22.957837Z shutdown complete |  +2023-03-12T13:13:11.498126Z starting(8.0.28) |  +2023-03-12T13:13:11.501941Z started(cluster) |  +2023-03-12T13:13:12.015863Z node3 joined |  +2023-03-12T13:13:12.015998Z | node2 joined +2023-03-12T13:13:12.020360Z | PRIMARY(n=2) +2023-03-12T13:13:12.515546Z CLOSED -> OPEN |  +2023-03-12T13:13:12.515641Z PRIMARY(n=2) |  +2023-03-12T13:13:12.516249Z OPEN -> PRIMARY |  +2023-03-12T13:13:13.245723Z will receive IST(seqno:170407338) |  +2023-03-12T13:13:13.247714Z node3 will resync local node |  +2023-03-12T13:13:13.247750Z PRIMARY -> JOINER |  +2023-03-12T13:13:13.248015Z |  local node will resync node2 +2023-03-12T13:13:13.248065Z |  SYNCED -> DONOR +2023-03-12T13:13:13.262238Z |  IST to node2(seqno:170407338) +2023-03-12T13:13:13.863959Z |  IST will be used +2023-03-12T13:13:14.886853Z got IST from node3 |  +2023-03-12T13:13:14.886942Z |  finished sending IST to node2 +2023-03-12T13:13:14.887000Z |  DESYNCED -> JOINED +2023-03-12T13:13:14.887249Z |  JOINED -> SYNCED +2023-03-12T13:13:19.031367Z wsrep recovery |  + +identifier node2 node3 +current path tests/logs/upgrade/node2.log tests/logs/upgrade/node3.log +last known ip 172.17.0.3 172.17.0.4 +last known name node2 node3 +mysql version 8.0.28 8.0.28 diff --git a/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_until_hiding_2_nodes b/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_until_hiding_2_nodes index 18c1cedc..063e3370 100644 --- a/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_until_hiding_2_nodes +++ b/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_all_until_hiding_2_nodes @@ -1,134 +1,138 @@ -identifier node2 -current path tests/logs/upgrade/node2.log -last known ip 172.17.0.3 -last known name node2 -mysql version 8.0.28 - -2023-03-12T07:24:13.733958Z starting(5.7.40) -2023-03-12T07:24:13.771126Z started(cluster) -2023-03-12T07:24:14.289375Z node1 joined -2023-03-12T07:24:14.289412Z node3 joined -2023-03-12T07:24:14.789002Z CLOSED -> OPEN -2023-03-12T07:24:14.789075Z PRIMARY(n=3) -2023-03-12T07:24:14.789560Z (restored)OPEN -> JOINED -2023-03-12T07:24:14.789785Z JOINED -> SYNCED -2023-03-12T07:34:47.289292Z received shutdown -2023-03-12T07:34:57.286990Z node1 joined -2023-03-12T07:34:57.287111Z node3 left -2023-03-12T07:34:57.290903Z node3 left -2023-03-12T07:35:12.293578Z PRIMARY(n=2) -2023-03-12T07:35:12.293705Z NON-PRIMARY(n=1) -2023-03-12T07:35:12.293723Z SYNCED -> OPEN -2023-03-12T07:35:12.293760Z OPEN -> CLOSED -2023-03-12T07:35:18.533851Z shutdown complete -2023-03-12T07:38:06.673334Z starting(5.7.40) -2023-03-12T07:38:06.680025Z started(cluster) -2023-03-12T07:38:06.681065Z safe_to_bootstrap: 1 -2023-03-12T07:38:06.693619Z bootstrapping -2023-03-12T07:38:06.695987Z CLOSED -> OPEN -2023-03-12T07:38:06.696042Z PRIMARY(n=1) -2023-03-12T07:38:06.696187Z (restored)OPEN -> JOINED -2023-03-12T07:38:06.696210Z JOINED -> SYNCED -2023-03-12T07:39:27.162350Z node3 joined -2023-03-12T07:39:27.164824Z PRIMARY(n=2) -2023-03-12T07:43:09.063375Z node1 joined -2023-03-12T07:43:09.063430Z node3 joined -2023-03-12T07:43:09.065740Z PRIMARY(n=3) -2023-03-12T07:49:45.317891Z received shutdown -2023-03-12T07:49:55.319157Z NON-PRIMARY(n=1) -2023-03-12T07:49:55.319203Z SYNCED -> OPEN -2023-03-12T07:49:55.319230Z OPEN -> CLOSED -2023-03-12T07:50:00.605309Z shutdown complete -2023-03-12T08:46:48.943442Z starting(5.7.40) -2023-03-12T08:46:48.947933Z started(cluster) -2023-03-12T08:46:48.992365Z node1 joined -2023-03-12T08:46:49.463255Z CLOSED -> OPEN -2023-03-12T08:46:49.463334Z PRIMARY(n=2) -2023-03-12T08:46:49.463988Z (restored)OPEN -> JOINED -2023-03-12T08:46:49.464124Z JOINED -> SYNCED -2023-03-12T08:48:28.470198Z node1 left -2023-03-12T08:48:28.477643Z node1 left -2023-03-12T08:48:28.477680Z PRIMARY(n=1) -2023-03-12T08:49:41.706020Z node1 joined -2023-03-12T08:49:41.713788Z PRIMARY(n=2) -2023-03-12T09:41:30.759927Z received shutdown -2023-03-12T09:41:41.775338Z NON-PRIMARY(n=1) -2023-03-12T09:41:41.775413Z SYNCED -> OPEN -2023-03-12T09:41:41.775442Z OPEN -> CLOSED -2023-03-12T09:41:48.745926Z shutdown complete - 5.7.40 - (version) -  V  - 8.0.28 -2023-03-12T09:55:30.928545Z starting(8.0.28) -2023-03-12T09:59:01.655066Z started(standalone) -2023-03-12T10:01:10.488475Z shutdown complete -2023-03-12T10:03:03.136053Z starting(8.0.28) -2023-03-12T10:03:03.139798Z started(cluster) -2023-03-12T10:03:03.157578Z not safe to bootstrap -2023-03-12T10:03:03.157601Z ABORTING -2023-03-12T10:03:03.157774Z shutdown complete -2023-03-12T10:03:03.163682Z CLOSED -> DESTROYED -2023-03-12T10:04:12.603100Z starting(8.0.28) -2023-03-12T10:04:12.608219Z started(cluster) -2023-03-12T10:04:12.609639Z safe_to_bootstrap: 1 -2023-03-12T10:04:12.623957Z bootstrapping -2023-03-12T10:04:12.628369Z CLOSED -> OPEN -2023-03-12T10:04:12.628477Z PRIMARY(n=1) -2023-03-12T10:04:12.628792Z (restored)OPEN -> JOINED -2023-03-12T10:04:12.628833Z JOINED -> SYNCED -2023-03-12T11:23:46.950430Z received shutdown -2023-03-12T11:23:56.953018Z SYNCED -> CLOSED -2023-03-12T11:24:03.294073Z shutdown complete -2023-03-12T11:24:33.315663Z starting(8.0.28) -2023-03-12T11:24:33.319800Z started(cluster) -2023-03-12T11:24:33.320989Z safe_to_bootstrap: 1 -2023-03-12T11:24:33.332251Z bootstrapping -2023-03-12T11:24:33.334384Z CLOSED -> OPEN -2023-03-12T11:24:33.334467Z PRIMARY(n=1) -2023-03-12T11:24:33.334699Z (restored)OPEN -> JOINED -2023-03-12T11:24:33.334761Z JOINED -> SYNCED -2023-03-12T11:35:14.693312Z node3 joined -2023-03-12T11:35:14.695410Z PRIMARY(n=2) -2023-03-12T11:35:16.321642Z SYNCED -> DONOR -2023-03-12T11:35:16.342707Z IST to node3(seqno:170403898) -2023-03-12T11:35:17.118100Z IST will be used -2023-03-12T11:35:18.140723Z finished sending IST to node3 -2023-03-12T11:35:18.140768Z DESYNCED -> JOINED -2023-03-12T11:35:18.141016Z JOINED -> SYNCED -2023-03-12T11:35:21.030164Z node3 left -2023-03-12T11:35:21.035732Z node3 left -2023-03-12T11:35:21.035794Z PRIMARY(n=1) -2023-03-12T11:39:20.681083Z node3 joined -2023-03-12T11:39:20.683800Z PRIMARY(n=2) -2023-03-12T11:39:21.948554Z SYNCED -> DONOR -2023-03-12T11:39:21.952242Z IST to node3(seqno:170403900) -2023-03-12T11:39:33.420743Z SST to node3 -2023-03-12T11:39:38.705565Z node3 left -2023-03-12T11:39:38.707686Z node3 left -2023-03-12T11:39:38.707695Z PRIMARY(n=1) -2023-03-12T11:39:38.734654Z SST error -2023-03-12T11:39:38.738833Z node2 failed to sync ??(node left) -2023-03-12T11:39:38.738842Z DESYNCED -> JOINED -2023-03-12T11:39:38.738942Z JOINED -> SYNCED -2023-03-12T12:22:48.704897Z received shutdown -2023-03-12T12:22:58.706338Z SYNCED -> CLOSED -2023-03-12T12:23:04.677082Z shutdown complete -2023-03-12T12:24:36.270274Z starting(8.0.28) -2023-03-12T12:24:36.274315Z started(cluster) -2023-03-12T12:24:36.275472Z safe_to_bootstrap: 1 -2023-03-12T12:24:36.287220Z bootstrapping -2023-03-12T12:24:36.290286Z CLOSED -> OPEN -2023-03-12T12:24:36.290365Z PRIMARY(n=1) -2023-03-12T12:24:36.290625Z (restored)OPEN -> JOINED -2023-03-12T12:24:36.290667Z JOINED -> SYNCED -2023-03-12T12:29:49.319032Z node1 joined -2023-03-12T12:29:49.323505Z PRIMARY(n=2) -2023-03-12T12:29:51.443525Z node1 left -2023-03-12T12:29:51.445280Z node1 left - -identifier node2 -current path tests/logs/upgrade/node2.log -last known ip 172.17.0.3 -last known name node2 -mysql version 8.0.28 +identifier node2 +current path tests/logs/upgrade/node2.log +last known ip 172.17.0.3 +last known name node2 +mysql version 8.0.28 + +2023-03-12T07:24:13.733958Z starting(5.7.40) +2023-03-12T07:24:13.771126Z started(cluster) +2023-03-12T07:24:14.289375Z node1 joined +2023-03-12T07:24:14.289412Z node3 joined +2023-03-12T07:24:14.789002Z CLOSED -> OPEN +2023-03-12T07:24:14.789075Z PRIMARY(n=3) +2023-03-12T07:24:14.789560Z (restored)OPEN -> JOINED +2023-03-12T07:24:14.789785Z JOINED -> SYNCED +2023-03-12T07:34:47.289292Z received shutdown +2023-03-12T07:34:57.286990Z node1 joined +2023-03-12T07:34:57.287111Z node3 left +2023-03-12T07:34:57.290903Z node3 left +2023-03-12T07:35:02.791416Z (repeated x17)node1 suspected to be down +2023-03-12T07:35:11.793101Z node1 suspected to be down +2023-03-12T07:35:12.293578Z PRIMARY(n=2) +2023-03-12T07:35:12.293705Z NON-PRIMARY(n=1) +2023-03-12T07:35:12.293723Z SYNCED -> OPEN +2023-03-12T07:35:12.293760Z OPEN -> CLOSED +2023-03-12T07:35:18.533851Z shutdown complete +2023-03-12T07:38:06.673334Z starting(5.7.40) +2023-03-12T07:38:06.680025Z started(cluster) +2023-03-12T07:38:06.681065Z safe_to_bootstrap: 1 +2023-03-12T07:38:06.693619Z bootstrapping +2023-03-12T07:38:06.695987Z CLOSED -> OPEN +2023-03-12T07:38:06.696042Z PRIMARY(n=1) +2023-03-12T07:38:06.696187Z (restored)OPEN -> JOINED +2023-03-12T07:38:06.696210Z JOINED -> SYNCED +2023-03-12T07:39:27.162350Z node3 joined +2023-03-12T07:39:27.164824Z PRIMARY(n=2) +2023-03-12T07:43:09.063375Z node1 joined +2023-03-12T07:43:09.063430Z node3 joined +2023-03-12T07:43:09.065740Z PRIMARY(n=3) +2023-03-12T07:49:45.317891Z received shutdown +2023-03-12T07:49:55.319157Z NON-PRIMARY(n=1) +2023-03-12T07:49:55.319203Z SYNCED -> OPEN +2023-03-12T07:49:55.319230Z OPEN -> CLOSED +2023-03-12T07:50:00.605309Z shutdown complete +2023-03-12T08:46:48.943442Z starting(5.7.40) +2023-03-12T08:46:48.947933Z started(cluster) +2023-03-12T08:46:48.992365Z node1 joined +2023-03-12T08:46:49.463255Z CLOSED -> OPEN +2023-03-12T08:46:49.463334Z PRIMARY(n=2) +2023-03-12T08:46:49.463988Z (restored)OPEN -> JOINED +2023-03-12T08:46:49.464124Z JOINED -> SYNCED +2023-03-12T08:48:28.470198Z node1 left +2023-03-12T08:48:28.477643Z node1 left +2023-03-12T08:48:28.477680Z PRIMARY(n=1) +2023-03-12T08:49:41.706020Z node1 joined +2023-03-12T08:49:41.713788Z PRIMARY(n=2) +2023-03-12T09:41:30.759927Z received shutdown +2023-03-12T09:41:41.775338Z NON-PRIMARY(n=1) +2023-03-12T09:41:41.775413Z SYNCED -> OPEN +2023-03-12T09:41:41.775442Z OPEN -> CLOSED +2023-03-12T09:41:48.745926Z shutdown complete + 5.7.40 + (version) +  V  + 8.0.28 +2023-03-12T09:55:30.928545Z starting(8.0.28) +2023-03-12T09:59:01.655066Z started(standalone) +2023-03-12T10:01:10.488475Z shutdown complete +2023-03-12T10:03:03.136053Z starting(8.0.28) +2023-03-12T10:03:03.139798Z started(cluster) +2023-03-12T10:03:03.157578Z not safe to bootstrap +2023-03-12T10:03:03.157601Z ABORTING +2023-03-12T10:03:03.157774Z shutdown complete +2023-03-12T10:03:03.163682Z CLOSED -> DESTROYED +2023-03-12T10:04:12.603100Z starting(8.0.28) +2023-03-12T10:04:12.608219Z started(cluster) +2023-03-12T10:04:12.609639Z safe_to_bootstrap: 1 +2023-03-12T10:04:12.623957Z bootstrapping +2023-03-12T10:04:12.628369Z CLOSED -> OPEN +2023-03-12T10:04:12.628477Z PRIMARY(n=1) +2023-03-12T10:04:12.628792Z (restored)OPEN -> JOINED +2023-03-12T10:04:12.628833Z JOINED -> SYNCED +2023-03-12T11:23:46.950430Z received shutdown +2023-03-12T11:23:56.953018Z SYNCED -> CLOSED +2023-03-12T11:24:03.294073Z shutdown complete +2023-03-12T11:24:33.315663Z starting(8.0.28) +2023-03-12T11:24:33.319800Z started(cluster) +2023-03-12T11:24:33.320989Z safe_to_bootstrap: 1 +2023-03-12T11:24:33.332251Z bootstrapping +2023-03-12T11:24:33.334384Z CLOSED -> OPEN +2023-03-12T11:24:33.334467Z PRIMARY(n=1) +2023-03-12T11:24:33.334699Z (restored)OPEN -> JOINED +2023-03-12T11:24:33.334761Z JOINED -> SYNCED +2023-03-12T11:35:14.693312Z node3 joined +2023-03-12T11:35:14.695410Z PRIMARY(n=2) +2023-03-12T11:35:16.321586Z local node will resync node3 +2023-03-12T11:35:16.321642Z SYNCED -> DONOR +2023-03-12T11:35:16.342707Z IST to node3(seqno:170403898) +2023-03-12T11:35:17.118100Z IST will be used +2023-03-12T11:35:18.140723Z finished sending IST to node3 +2023-03-12T11:35:18.140768Z DESYNCED -> JOINED +2023-03-12T11:35:18.141016Z JOINED -> SYNCED +2023-03-12T11:35:21.030164Z node3 left +2023-03-12T11:35:21.035732Z node3 left +2023-03-12T11:35:21.035794Z PRIMARY(n=1) +2023-03-12T11:39:20.681083Z node3 joined +2023-03-12T11:39:20.683800Z PRIMARY(n=2) +2023-03-12T11:39:21.948501Z local node will resync node3 +2023-03-12T11:39:21.948554Z SYNCED -> DONOR +2023-03-12T11:39:21.952242Z IST to node3(seqno:170403900) +2023-03-12T11:39:33.420743Z SST to node3 +2023-03-12T11:39:38.705565Z node3 left +2023-03-12T11:39:38.707686Z node3 left +2023-03-12T11:39:38.707695Z PRIMARY(n=1) +2023-03-12T11:39:38.734654Z SST error +2023-03-12T11:39:38.738833Z node2 failed to sync ??(node left) +2023-03-12T11:39:38.738842Z DESYNCED -> JOINED +2023-03-12T11:39:38.738942Z JOINED -> SYNCED +2023-03-12T12:22:48.704897Z received shutdown +2023-03-12T12:22:58.706338Z SYNCED -> CLOSED +2023-03-12T12:23:04.677082Z shutdown complete +2023-03-12T12:24:36.270274Z starting(8.0.28) +2023-03-12T12:24:36.274315Z started(cluster) +2023-03-12T12:24:36.275472Z safe_to_bootstrap: 1 +2023-03-12T12:24:36.287220Z bootstrapping +2023-03-12T12:24:36.290286Z CLOSED -> OPEN +2023-03-12T12:24:36.290365Z PRIMARY(n=1) +2023-03-12T12:24:36.290625Z (restored)OPEN -> JOINED +2023-03-12T12:24:36.290667Z JOINED -> SYNCED +2023-03-12T12:29:49.319032Z node1 joined +2023-03-12T12:29:49.323505Z PRIMARY(n=2) +2023-03-12T12:29:51.443525Z node1 left +2023-03-12T12:29:51.445280Z node1 left + +identifier node2 +current path tests/logs/upgrade/node2.log +last known ip 172.17.0.3 +last known name node2 +mysql version 8.0.28 diff --git a/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_sst b/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_sst index 122fb03a..97da1dd5 100644 --- a/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_sst +++ b/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_sst @@ -8,31 +8,42 @@ mysql version 8.0.28 8.0.28 (version)  V  8.0.28 +2023-03-12T11:35:16.321586Z | local node will resync node3 | 2023-03-12T11:35:16.342707Z | IST to node3(seqno:170403898) | 2023-03-12T11:35:17.118100Z | IST will be used | 2023-03-12T11:35:18.140723Z | finished sending IST to node3 | +2023-03-12T11:39:21.948501Z | local node will resync node3 | 2023-03-12T11:39:21.952242Z | IST to node3(seqno:170403900) | 2023-03-12T11:39:33.420743Z | SST to node3 | 2023-03-12T11:39:38.734654Z | SST error | 2023-03-12T11:39:38.738833Z | node2 failed to sync ??(node left) | 2023-03-12T12:48:44.597299Z | |  will receive IST(seqno:170403905) +2023-03-12T12:48:44.599287Z | local node will resync node3 |  +2023-03-12T12:48:44.599346Z | |  node2 will resync local node 2023-03-12T12:48:44.616436Z | IST to node3(seqno:170403905) |  2023-03-12T12:48:45.044873Z | IST will be used |  2023-03-12T12:48:46.064764Z | finished sending IST to node3 |  2023-03-12T12:48:46.065014Z | | got IST from node2 2023-03-12T12:48:54.269978Z | |  IST received(seqno:170403905) +2023-03-12T13:04:25.731994Z | node3 will resync node1 |  +2023-03-12T13:04:25.732124Z | |  local node will resync node1 2023-03-12T13:04:25.735999Z | |  IST to node1(seqno:170407335) 2023-03-12T13:04:37.415791Z | |  SST to node1 2023-03-12T13:04:39.715275Z | |  SST error 2023-03-12T13:04:39.720325Z | node3 failed to sync ??(node left) |  2023-03-12T13:04:39.720379Z | |  node3 failed to sync ??(node left) 2023-03-12T13:13:13.245723Z | will receive IST(seqno:170407338) |  +2023-03-12T13:13:13.247714Z | node3 will resync local node |  +2023-03-12T13:13:13.248015Z | |  local node will resync node2 2023-03-12T13:13:13.262238Z | |  IST to node2(seqno:170407338) 2023-03-12T13:13:13.863959Z | |  IST will be used 2023-03-12T13:13:14.886853Z | got IST from node3 |  2023-03-12T13:13:14.886942Z | |  finished sending IST to node2 2023-03-12T13:13:19.156722Z | IST received(seqno:170407338) |  2023-03-12T19:35:07.638676Z will receive IST(seqno:178226774) |  |  +2023-03-12T19:35:07.644560Z |  |  local node will resync node1 +2023-03-12T19:35:07.644668Z node3 will resync node1 |  |  +2023-03-12T19:35:07.644740Z |  node3 will resync node1 |  2023-03-12T19:36:48.567087Z timeout from donor in gtid/keyring stage |  |  2023-03-12T19:36:48.589084Z SST error |  |  2023-03-12T19:36:48.590647Z former SST cancelled |  |  diff --git a/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_views b/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_views index ef3a976c..c3a6cddf 100644 --- a/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_views +++ b/src/go/pt-galera-log-explainer/tests/expected/upgrade_list_views @@ -1,139 +1,141 @@ -identifier 172.17.0.2 node2 node3 -current path tests/logs/upgrade/node1.log tests/logs/upgrade/node2.log tests/logs/upgrade/node3.log -last known ip 172.17.0.2 172.17.0.3 172.17.0.4 -last known name node2 node3 -mysql version 8.0.28 8.0.28 8.0.28 - -2023-03-12T07:24:14.289375Z | node1 joined | -2023-03-12T07:24:14.289412Z | node3 joined | -2023-03-12T07:24:14.789075Z | PRIMARY(n=3) | -2023-03-12T07:34:57.286990Z | node1 joined | -2023-03-12T07:34:57.287111Z | node3 left | -2023-03-12T07:34:57.290903Z | node3 left | -2023-03-12T07:35:12.293578Z | PRIMARY(n=2) | -2023-03-12T07:35:12.293705Z | NON-PRIMARY(n=1) | -2023-03-12T07:38:06.681065Z | safe_to_bootstrap: 1 | -2023-03-12T07:38:06.693619Z | bootstrapping | -2023-03-12T07:38:06.696042Z | PRIMARY(n=1) | -2023-03-12T07:39:27.162350Z | node3 joined | -2023-03-12T07:39:27.164824Z | PRIMARY(n=2) | -2023-03-12T07:43:09.063375Z | node1 joined | -2023-03-12T07:43:09.063430Z | node3 joined | -2023-03-12T07:43:09.065740Z | PRIMARY(n=3) | -2023-03-12T07:49:55.319157Z | NON-PRIMARY(n=1) | -2023-03-12T08:46:48.992365Z | node1 joined | -2023-03-12T08:46:49.463334Z | PRIMARY(n=2) | -2023-03-12T08:48:28.470198Z | node1 left | -2023-03-12T08:48:28.477643Z | node1 left | -2023-03-12T08:48:28.477680Z | PRIMARY(n=1) | -2023-03-12T08:49:41.706020Z | node1 joined | -2023-03-12T08:49:41.713788Z | PRIMARY(n=2) | -2023-03-12T09:41:41.775338Z | NON-PRIMARY(n=1) | - 5.7.40 - (version) -  V  - 8.0.28 -2023-03-12T10:03:03.157578Z | not safe to bootstrap | -2023-03-12T10:04:12.609639Z | safe_to_bootstrap: 1 | -2023-03-12T10:04:12.623957Z | bootstrapping | -2023-03-12T10:04:12.628477Z | PRIMARY(n=1) | -2023-03-12T11:24:33.320989Z | safe_to_bootstrap: 1 | -2023-03-12T11:24:33.332251Z | bootstrapping | -2023-03-12T11:24:33.334467Z | PRIMARY(n=1) | -2023-03-12T11:35:14.693312Z | node3 joined | -2023-03-12T11:35:14.695410Z | PRIMARY(n=2) | -2023-03-12T11:35:21.030164Z | node3 left | -2023-03-12T11:35:21.035732Z | node3 left | -2023-03-12T11:35:21.035794Z | PRIMARY(n=1) | -2023-03-12T11:39:20.681083Z | node3 joined | -2023-03-12T11:39:20.683800Z | PRIMARY(n=2) | -2023-03-12T11:39:38.705565Z | node3 left | -2023-03-12T11:39:38.707686Z | node3 left | -2023-03-12T11:39:38.707695Z | PRIMARY(n=1) | -2023-03-12T12:24:36.275472Z | safe_to_bootstrap: 1 | -2023-03-12T12:24:36.287220Z | bootstrapping | -2023-03-12T12:24:36.290365Z | PRIMARY(n=1) | -2023-03-12T12:29:49.319032Z | node1 joined | -2023-03-12T12:29:49.323505Z | PRIMARY(n=2) | -2023-03-12T12:29:51.443525Z | node1 left | -2023-03-12T12:29:51.445280Z | node1 left | -2023-03-12T12:29:51.445300Z | PRIMARY(n=1) | -2023-03-12T12:48:43.521685Z | node3 joined | -2023-03-12T12:48:43.521846Z | |  node2 joined -2023-03-12T12:48:43.526717Z | PRIMARY(n=2) | -2023-03-12T12:48:43.820929Z | |  PRIMARY(n=2) -2023-03-12T13:04:24.476576Z | node3 joined |  -2023-03-12T13:04:24.476642Z | node1 joined |  -2023-03-12T13:04:24.476806Z | |  node1 joined -2023-03-12T13:04:24.476863Z | |  node2 joined -2023-03-12T13:04:24.478964Z | PRIMARY(n=3) |  -2023-03-12T13:04:24.479206Z | |  PRIMARY(n=3) -2023-03-12T13:04:38.645597Z | node3 joined |  -2023-03-12T13:04:38.645710Z | node1 left |  -2023-03-12T13:04:38.647921Z | |  node2 joined -2023-03-12T13:04:38.647981Z | |  node1 left -2023-03-12T13:04:38.650097Z | |  node1 left -2023-03-12T13:04:38.650125Z | |  PRIMARY(n=2) -2023-03-12T13:04:38.652812Z | node1 left |  -2023-03-12T13:04:38.652875Z | PRIMARY(n=2) |  -2023-03-12T13:12:13.679070Z | |  node2 left -2023-03-12T13:12:13.681813Z | |  node2 left -2023-03-12T13:12:13.681867Z | |  PRIMARY(n=1) -2023-03-12T13:12:13.682286Z | NON-PRIMARY(n=1) |  -2023-03-12T13:13:12.015863Z | node3 joined |  -2023-03-12T13:13:12.015998Z | | node2 joined -2023-03-12T13:13:12.020360Z | | PRIMARY(n=2) -2023-03-12T13:13:12.515641Z | PRIMARY(n=2) |  -2023-03-12T19:35:06.375917Z | |  node2 joined -2023-03-12T19:35:06.375974Z | |  node1 joined -2023-03-12T19:35:06.376012Z node3 joined |  |  -2023-03-12T19:35:06.376016Z | node3 joined |  -2023-03-12T19:35:06.376026Z node2 joined |  |  -2023-03-12T19:35:06.376081Z | node1 joined |  -2023-03-12T19:35:06.383186Z | PRIMARY(n=3) |  -2023-03-12T19:35:06.385445Z | |  PRIMARY(n=3) -2023-03-12T19:35:06.875717Z PRIMARY(n=3) |  |  -2023-03-12T19:36:48.590054Z |  |  node2 joined -2023-03-12T19:36:48.590121Z |  |  node1 left -2023-03-12T19:36:48.590280Z |  node3 joined |  -2023-03-12T19:36:48.590338Z NON-PRIMARY(n=1) |  |  -2023-03-12T19:36:48.590388Z |  node1 left |  -2023-03-12T19:36:48.597786Z |  |  node1 left -2023-03-12T19:36:48.597826Z |  |  PRIMARY(n=2) -2023-03-12T19:36:48.604279Z |  node1 left |  -2023-03-12T19:36:48.604341Z |  PRIMARY(n=2) |  -2023-03-12T19:43:17.630191Z | node3 joined |  -2023-03-12T19:43:17.630208Z node3 joined |  |  -2023-03-12T19:43:17.630221Z node2 joined |  |  -2023-03-12T19:43:17.630243Z | node1 joined |  -2023-03-12T19:43:17.634138Z | |  node2 joined -2023-03-12T19:43:17.634229Z | |  node1 joined -2023-03-12T19:43:17.643210Z | PRIMARY(n=3) |  -2023-03-12T19:43:17.648163Z | |  PRIMARY(n=3) -2023-03-12T19:43:18.130230Z PRIMARY(n=3) |  |  -2023-03-12T19:44:59.840669Z | |  node2 joined -2023-03-12T19:44:59.840745Z | |  node1 left -2023-03-12T19:44:59.840933Z | node3 joined |  -2023-03-12T19:44:59.841034Z | node1 left |  -2023-03-12T19:44:59.841189Z NON-PRIMARY(n=1) |  |  -2023-03-12T19:44:59.848349Z |  |  node1 left -2023-03-12T19:44:59.848409Z |  |  PRIMARY(n=2) -2023-03-12T19:44:59.855443Z |  node1 left |  -2023-03-12T19:44:59.855491Z |  PRIMARY(n=2) |  -2023-03-12T21:55:59.918448Z |  |  node2 left -2023-03-12T21:55:59.924796Z |  |  node2 left -2023-03-12T21:55:59.924897Z |  |  PRIMARY(n=1) -2023-03-12T21:55:59.925551Z |  NON-PRIMARY(n=1) |  -2023-03-12T21:58:44.885014Z |  | node2 joined -2023-03-12T21:58:44.885179Z |  node3 joined |  -2023-03-12T21:58:44.887985Z |  | PRIMARY(n=2) -2023-03-12T21:58:45.384861Z |  PRIMARY(n=2) |  -2023-03-12T22:00:28.090598Z |  |  node2 left -2023-03-12T22:00:28.094664Z |  |  node2 left -2023-03-12T22:00:28.094708Z |  |  PRIMARY(n=1) - -identifier 172.17.0.2 node2 node3 -current path tests/logs/upgrade/node1.log tests/logs/upgrade/node2.log tests/logs/upgrade/node3.log -last known ip 172.17.0.2 172.17.0.3 172.17.0.4 -last known name node2 node3 -mysql version 8.0.28 8.0.28 8.0.28 +identifier 172.17.0.2 node2 node3 +current path tests/logs/upgrade/node1.log tests/logs/upgrade/node2.log tests/logs/upgrade/node3.log +last known ip 172.17.0.2 172.17.0.3 172.17.0.4 +last known name node2 node3 +mysql version 8.0.28 8.0.28 8.0.28 + +2023-03-12T07:24:14.289375Z | node1 joined | +2023-03-12T07:24:14.289412Z | node3 joined | +2023-03-12T07:24:14.789075Z | PRIMARY(n=3) | +2023-03-12T07:34:57.286990Z | node1 joined | +2023-03-12T07:34:57.287111Z | node3 left | +2023-03-12T07:34:57.290903Z | node3 left | +2023-03-12T07:35:02.791416Z | (repeated x17)node1 suspected to be down | +2023-03-12T07:35:11.793101Z | node1 suspected to be down | +2023-03-12T07:35:12.293578Z | PRIMARY(n=2) | +2023-03-12T07:35:12.293705Z | NON-PRIMARY(n=1) | +2023-03-12T07:38:06.681065Z | safe_to_bootstrap: 1 | +2023-03-12T07:38:06.693619Z | bootstrapping | +2023-03-12T07:38:06.696042Z | PRIMARY(n=1) | +2023-03-12T07:39:27.162350Z | node3 joined | +2023-03-12T07:39:27.164824Z | PRIMARY(n=2) | +2023-03-12T07:43:09.063375Z | node1 joined | +2023-03-12T07:43:09.063430Z | node3 joined | +2023-03-12T07:43:09.065740Z | PRIMARY(n=3) | +2023-03-12T07:49:55.319157Z | NON-PRIMARY(n=1) | +2023-03-12T08:46:48.992365Z | node1 joined | +2023-03-12T08:46:49.463334Z | PRIMARY(n=2) | +2023-03-12T08:48:28.470198Z | node1 left | +2023-03-12T08:48:28.477643Z | node1 left | +2023-03-12T08:48:28.477680Z | PRIMARY(n=1) | +2023-03-12T08:49:41.706020Z | node1 joined | +2023-03-12T08:49:41.713788Z | PRIMARY(n=2) | +2023-03-12T09:41:41.775338Z | NON-PRIMARY(n=1) | + 5.7.40 + (version) +  V  + 8.0.28 +2023-03-12T10:03:03.157578Z | not safe to bootstrap | +2023-03-12T10:04:12.609639Z | safe_to_bootstrap: 1 | +2023-03-12T10:04:12.623957Z | bootstrapping | +2023-03-12T10:04:12.628477Z | PRIMARY(n=1) | +2023-03-12T11:24:33.320989Z | safe_to_bootstrap: 1 | +2023-03-12T11:24:33.332251Z | bootstrapping | +2023-03-12T11:24:33.334467Z | PRIMARY(n=1) | +2023-03-12T11:35:14.693312Z | node3 joined | +2023-03-12T11:35:14.695410Z | PRIMARY(n=2) | +2023-03-12T11:35:21.030164Z | node3 left | +2023-03-12T11:35:21.035732Z | node3 left | +2023-03-12T11:35:21.035794Z | PRIMARY(n=1) | +2023-03-12T11:39:20.681083Z | node3 joined | +2023-03-12T11:39:20.683800Z | PRIMARY(n=2) | +2023-03-12T11:39:38.705565Z | node3 left | +2023-03-12T11:39:38.707686Z | node3 left | +2023-03-12T11:39:38.707695Z | PRIMARY(n=1) | +2023-03-12T12:24:36.275472Z | safe_to_bootstrap: 1 | +2023-03-12T12:24:36.287220Z | bootstrapping | +2023-03-12T12:24:36.290365Z | PRIMARY(n=1) | +2023-03-12T12:29:49.319032Z | node1 joined | +2023-03-12T12:29:49.323505Z | PRIMARY(n=2) | +2023-03-12T12:29:51.443525Z | node1 left | +2023-03-12T12:29:51.445280Z | node1 left | +2023-03-12T12:29:51.445300Z | PRIMARY(n=1) | +2023-03-12T12:48:43.521685Z | node3 joined | +2023-03-12T12:48:43.521846Z | |  node2 joined +2023-03-12T12:48:43.526717Z | PRIMARY(n=2) | +2023-03-12T12:48:43.820929Z | |  PRIMARY(n=2) +2023-03-12T13:04:24.476576Z | node3 joined |  +2023-03-12T13:04:24.476642Z | node1 joined |  +2023-03-12T13:04:24.476806Z | |  node1 joined +2023-03-12T13:04:24.476863Z | |  node2 joined +2023-03-12T13:04:24.478964Z | PRIMARY(n=3) |  +2023-03-12T13:04:24.479206Z | |  PRIMARY(n=3) +2023-03-12T13:04:38.645597Z | node3 joined |  +2023-03-12T13:04:38.645710Z | node1 left |  +2023-03-12T13:04:38.647921Z | |  node2 joined +2023-03-12T13:04:38.647981Z | |  node1 left +2023-03-12T13:04:38.650097Z | |  node1 left +2023-03-12T13:04:38.650125Z | |  PRIMARY(n=2) +2023-03-12T13:04:38.652812Z | node1 left |  +2023-03-12T13:04:38.652875Z | PRIMARY(n=2) |  +2023-03-12T13:12:13.679070Z | |  node2 left +2023-03-12T13:12:13.681813Z | |  node2 left +2023-03-12T13:12:13.681867Z | |  PRIMARY(n=1) +2023-03-12T13:12:13.682286Z | NON-PRIMARY(n=1) |  +2023-03-12T13:13:12.015863Z | node3 joined |  +2023-03-12T13:13:12.015998Z | | node2 joined +2023-03-12T13:13:12.020360Z | | PRIMARY(n=2) +2023-03-12T13:13:12.515641Z | PRIMARY(n=2) |  +2023-03-12T19:35:06.375917Z | |  node2 joined +2023-03-12T19:35:06.375974Z | |  node1 joined +2023-03-12T19:35:06.376012Z node3 joined |  |  +2023-03-12T19:35:06.376016Z | node3 joined |  +2023-03-12T19:35:06.376026Z node2 joined |  |  +2023-03-12T19:35:06.376081Z | node1 joined |  +2023-03-12T19:35:06.383186Z | PRIMARY(n=3) |  +2023-03-12T19:35:06.385445Z | |  PRIMARY(n=3) +2023-03-12T19:35:06.875717Z PRIMARY(n=3) |  |  +2023-03-12T19:36:48.590054Z |  |  node2 joined +2023-03-12T19:36:48.590121Z |  |  node1 left +2023-03-12T19:36:48.590280Z |  node3 joined |  +2023-03-12T19:36:48.590338Z NON-PRIMARY(n=1) |  |  +2023-03-12T19:36:48.590388Z |  node1 left |  +2023-03-12T19:36:48.597786Z |  |  node1 left +2023-03-12T19:36:48.597826Z |  |  PRIMARY(n=2) +2023-03-12T19:36:48.604279Z |  node1 left |  +2023-03-12T19:36:48.604341Z |  PRIMARY(n=2) |  +2023-03-12T19:43:17.630191Z | node3 joined |  +2023-03-12T19:43:17.630208Z node3 joined |  |  +2023-03-12T19:43:17.630221Z node2 joined |  |  +2023-03-12T19:43:17.630243Z | node1 joined |  +2023-03-12T19:43:17.634138Z | |  node2 joined +2023-03-12T19:43:17.634229Z | |  node1 joined +2023-03-12T19:43:17.643210Z | PRIMARY(n=3) |  +2023-03-12T19:43:17.648163Z | |  PRIMARY(n=3) +2023-03-12T19:43:18.130230Z PRIMARY(n=3) |  |  +2023-03-12T19:44:59.840669Z | |  node2 joined +2023-03-12T19:44:59.840745Z | |  node1 left +2023-03-12T19:44:59.840933Z | node3 joined |  +2023-03-12T19:44:59.841034Z | node1 left |  +2023-03-12T19:44:59.841189Z NON-PRIMARY(n=1) |  |  +2023-03-12T19:44:59.848349Z |  |  node1 left +2023-03-12T19:44:59.848409Z |  |  PRIMARY(n=2) +2023-03-12T19:44:59.855443Z |  node1 left |  +2023-03-12T19:44:59.855491Z |  PRIMARY(n=2) |  +2023-03-12T21:55:59.918448Z |  |  node2 left +2023-03-12T21:55:59.924796Z |  |  node2 left +2023-03-12T21:55:59.924897Z |  |  PRIMARY(n=1) +2023-03-12T21:55:59.925551Z |  NON-PRIMARY(n=1) |  +2023-03-12T21:58:44.885014Z |  | node2 joined +2023-03-12T21:58:44.885179Z |  node3 joined |  +2023-03-12T21:58:44.887985Z |  | PRIMARY(n=2) +2023-03-12T21:58:45.384861Z |  PRIMARY(n=2) |  +2023-03-12T22:00:28.090598Z |  |  node2 left +2023-03-12T22:00:28.094664Z |  |  node2 left +2023-03-12T22:00:28.094708Z |  |  PRIMARY(n=1) + +identifier 172.17.0.2 node2 node3 +current path tests/logs/upgrade/node1.log tests/logs/upgrade/node2.log tests/logs/upgrade/node3.log +last known ip 172.17.0.2 172.17.0.3 172.17.0.4 +last known name node2 node3 +mysql version 8.0.28 8.0.28 8.0.28 diff --git a/src/go/pt-galera-log-explainer/types/loginfo.go b/src/go/pt-galera-log-explainer/types/loginfo.go index a31e110c..81322d5b 100644 --- a/src/go/pt-galera-log-explainer/types/loginfo.go +++ b/src/go/pt-galera-log-explainer/types/loginfo.go @@ -11,8 +11,6 @@ type Verbosity int const ( Info Verbosity = iota - // Detailed is having every suspect/warn - Detailed // DebugMySQL only includes finding that are usually not relevant to show but useful to create the log context (eg: how we found the local address) DebugMySQL Debug