diff --git a/src/go/pt-galera-log-explainer/detect_operator.go b/src/go/pt-galera-log-explainer/detect_operator.go new file mode 100644 index 00000000..d9639597 --- /dev/null +++ b/src/go/pt-galera-log-explainer/detect_operator.go @@ -0,0 +1,23 @@ +package main + +import ( + "os/exec" + + "github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/types" + "github.com/rs/zerolog/log" +) + +// areOperatorFiles will assume every files are from k8s if one is found +func areOperatorFiles(paths []string) bool { + + for _, path := range paths { + + cmd := exec.Command(CLI.GrepCmd, "-q", "-a", "-m", "1", "^"+types.OperatorLogPrefix, path) + err := cmd.Run() + if err == nil { + return true + } + log.Debug().Err(err).Str("path", path).Msg("operator detection result") + } + return false +} diff --git a/src/go/pt-galera-log-explainer/internal.go b/src/go/pt-galera-log-explainer/internal.go index efeff999..0ba677db 100644 --- a/src/go/pt-galera-log-explainer/internal.go +++ b/src/go/pt-galera-log-explainer/internal.go @@ -95,7 +95,7 @@ func prepareGrepArgument(regexes types.RegexMap) string { // I'm not adding pxcoperator map the same way others are used, because they do not have the same formats and same place // it needs to be put on the front so that it's not 'merged' with the '{"log":"' json prefix // this is to keep things as close as '^' as possible to keep doing prefix searches - grepRegex += "((" + strings.Join(regex.PXCOperatorMap.Compile(), "|") + ")|^{\"log\":\"" + grepRegex += "((" + strings.Join(regex.PXCOperatorMap.Compile(), "|") + ")|^" + types.OperatorLogPrefix regexes.Merge(regex.PXCOperatorMap) } if CLI.Since != nil { diff --git a/src/go/pt-galera-log-explainer/main.go b/src/go/pt-galera-log-explainer/main.go index 2fa57200..888e8e5d 100644 --- a/src/go/pt-galera-log-explainer/main.go +++ b/src/go/pt-galera-log-explainer/main.go @@ -60,13 +60,29 @@ func main() { ) zerolog.TimeFieldFormat = zerolog.TimeFormatUnix - zerolog.SetGlobalLevel(zerolog.WarnLevel) - log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) + zerolog.SetGlobalLevel(zerolog.InfoLevel) + log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr, NoColor: CLI.NoColor, FormatTimestamp: func(_ interface{}) string { return "" }}) if CLI.Verbosity == types.Debug { zerolog.SetGlobalLevel(zerolog.DebugLevel) } utils.SkipColor = CLI.NoColor + + var paths []string + switch kongcli.Command() { + case "list ": + paths = CLI.List.Paths + case "ctx ": + paths = CLI.Ctx.Paths + case "conflicts ": + paths = CLI.Conflicts.Paths + } + + if !CLI.PxcOperator && areOperatorFiles(paths) { + CLI.PxcOperator = true + log.Info().Msg("Detected logs coming from Percona XtraDB Cluster Operator, enabling --pxc-operator") + } + translate.AssumeIPStable = !CLI.PxcOperator err := kongcli.Run() diff --git a/src/go/pt-galera-log-explainer/main_test.go b/src/go/pt-galera-log-explainer/main_test.go index db1661a1..f0335286 100644 --- a/src/go/pt-galera-log-explainer/main_test.go +++ b/src/go/pt-galera-log-explainer/main_test.go @@ -104,6 +104,12 @@ func TestMain(t *testing.T) { path: "tests/logs/operator_split/*", }, + { + name: "operator_auto_ambiguous_ips_list_all_no_color", + cmd: []string{"list", "--all", "--no-color"}, + path: "tests/logs/operator_ambiguous_ips/*", + }, + { name: "conflict_conflicts", cmd: []string{"conflicts"}, diff --git a/src/go/pt-galera-log-explainer/regex/date.go b/src/go/pt-galera-log-explainer/regex/date.go index 1d7f8fed..799895fb 100644 --- a/src/go/pt-galera-log-explainer/regex/date.go +++ b/src/go/pt-galera-log-explainer/regex/date.go @@ -5,6 +5,7 @@ import ( "strconv" "time" + "github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/types" "github.com/percona/percona-toolkit/src/go/pt-galera-log-explainer/utils" "github.com/rs/zerolog/log" ) @@ -93,11 +94,9 @@ func NoDatesRegex(skipLeadingCircumflex bool) string { return "^(?![0-9]{4})" } -const k8sprefix = `{"log":"` - func SearchDateFromLog(logline string) (time.Time, string, bool) { - if logline[:len(k8sprefix)] == k8sprefix { - logline = logline[len(k8sprefix):] + if logline[:len(types.OperatorLogPrefix)] == types.OperatorLogPrefix { + logline = logline[len(types.OperatorLogPrefix):] } for _, layout := range DateLayouts { if len(logline) < len(layout) { diff --git a/src/go/pt-galera-log-explainer/regex/operator.go b/src/go/pt-galera-log-explainer/regex/operator.go index fd72c8d5..4d613be1 100644 --- a/src/go/pt-galera-log-explainer/regex/operator.go +++ b/src/go/pt-galera-log-explainer/regex/operator.go @@ -50,7 +50,7 @@ var PXCOperatorMap = types.RegexMap{ "RegexGcacheScan": &types.LogRegex{ // those "operators" regexes do not have the log prefix added implicitly. It's not strictly needed, but // it will help to avoid catching random piece of log out of order - Regex: regexp.MustCompile(k8sprefix + ".*GCache::RingBuffer initial scan"), + Regex: regexp.MustCompile(types.OperatorLogPrefix + ".*GCache::RingBuffer initial scan"), Handler: func(submatches map[string]string, logCtx types.LogCtx, log string, date time.Time) (types.LogCtx, types.LogDisplayer) { return logCtx, types.SimpleDisplayer("recovering gcache") }, diff --git a/src/go/pt-galera-log-explainer/tests/expected/operator_auto_ambiguous_ips_list_all_no_color b/src/go/pt-galera-log-explainer/tests/expected/operator_auto_ambiguous_ips_list_all_no_color new file mode 100644 index 00000000..46909326 --- /dev/null +++ b/src/go/pt-galera-log-explainer/tests/expected/operator_auto_ambiguous_ips_list_all_no_color @@ -0,0 +1,2331 @@ +INF Detected logs coming from Percona XtraDB Cluster Operator, enabling --pxc-operator +identifier cluster1-0 cluster1-1 cluster1-2 +current path tests/logs/operator_ambiguous_ips/node1.log tests/logs/operator_ambiguous_ips/node2.log tests/logs/operator_ambiguous_ips/node3.log +last known ip 10.16.29.34 10.16.27.98 10.16.28.213 +last known name cluster1-0 cluster1-1 cluster1-2 +mysql version 8.0.31 8.0.31 8.0.31 + +2023-05-10T09:06:21.282700Z | starting(8.0.31) | +2023-05-10T09:06:21.286713Z | started(cluster) | +2023-05-10T09:06:21.287690Z | no grastate.dat file | +2023-05-10T09:06:21.287698Z | bootstrapping(empty grastate) | +2023-05-10T09:06:21.287729Z | safe_to_bootstrap: 1 | + 10.16.27.98 + (node ip) + V + 10.16.27.195 +2023-05-10T09:06:21.805629Z | cluster1-0 joined | +2023-05-10T09:06:22.304843Z | CLOSED -> OPEN | +2023-05-10T09:06:22.305137Z | PRIMARY(n=2) | +2023-05-10T09:06:22.306346Z | OPEN -> PRIMARY | +2023-05-10T09:06:22.908357Z | will receive SST | +2023-05-10T09:06:22.909183Z | cluster1-0 will resync local node | +2023-05-10T09:06:22.909264Z | PRIMARY -> JOINER | +2023-05-10T09:06:23.728522Z | receiving SST | +2023-05-10T09:41:07.257241Z | cluster1-0 failed to sync cluster1-1 | +2023-05-10T09:41:07.257343Z | will never receive SST, aborting | +2023-05-10T09:41:08.259656Z | terminated | +2023-05-10T09:41:08.259664Z | former SST cancelled | +2023-05-10T09:41:08Z | crash: got signal 11 | +2023-05-10T09:41:48.083981Z | starting(8.0.31) | +2023-05-10T09:41:48.086499Z | started(cluster) | +2023-05-10T09:41:48.087403Z | safe_to_bootstrap: 1 | +2023-05-10T09:41:48.087529Z | recovering gcache | +2023-05-10T09:41:57.935555Z | recovering gcache | +2023-05-10T09:42:20.265286Z | cluster1-0 joined | +2023-05-10T09:42:20.592086Z | CLOSED -> OPEN | +2023-05-10T09:42:20.592348Z | PRIMARY(n=2) | +2023-05-10T09:42:20.594549Z | OPEN -> PRIMARY | +2023-05-10T09:42:21.259393Z | will receive SST | +2023-05-10T09:42:21.260025Z | cluster1-0 will resync local node | +2023-05-10T09:42:21.260093Z | PRIMARY -> JOINER | +2023-05-10T09:42:22.066358Z | receiving SST | +2023-05-10T10:12:45.191674Z | 10.16.26.202 joined | +2023-05-10T10:12:45.191779Z | cluster1-0 joined | +2023-05-10T10:12:45.193317Z | PRIMARY(n=3) | +2023-05-10T10:12:46.370533Z | cluster1-2 cannot find donor | +2023-05-10T10:12:47.371783Z | cluster1-2 cannot find donor | +2023-05-10T10:12:48.373115Z | (repeated x85)cluster1-2 cannot find donor | +2023-05-10T10:14:14.479756Z | cluster1-2 cannot find donor | +2023-05-10T10:14:14.784317Z | SST error | +2023-05-10T10:14:14.786328Z | NON-PRIMARY(n=1) | +2023-05-10T10:14:14.786496Z | JOINER -> OPEN | +2023-05-10T10:14:14.786591Z | OPEN -> CLOSED | +2023-05-10T10:14:14.786750Z | terminated | +2023-05-10T10:14:14.786767Z | former SST cancelled | +2023-05-10T10:14:14Z | crash: got signal 11 | +2023-05-10T10:48:43.442930Z | starting(8.0.31) | +2023-05-10T10:48:43.445605Z | started(cluster) | +2023-05-10T10:48:43.446512Z | safe_to_bootstrap: 1 | +2023-05-10T10:48:43.446635Z | recovering gcache | +2023-05-10T10:48:53.296174Z | recovering gcache | + 10.16.27.195 + (node ip) + V + 10.16.27.93 +2023-05-10T10:49:16.479739Z | 10.16.26.202 joined | +2023-05-10T10:49:16.479769Z | cluster1-0 joined | +2023-05-10T10:49:16.978527Z | CLOSED -> OPEN | +2023-05-10T10:49:16.978767Z | PRIMARY(n=3) | +2023-05-10T10:49:16.980011Z | OPEN -> PRIMARY | +2023-05-10T10:49:17.553352Z | will receive SST | +2023-05-10T10:49:17.554588Z | cluster1-0 will resync local node | +2023-05-10T10:49:17.554657Z | PRIMARY -> JOINER | +2023-05-10T10:49:18.376721Z | receiving SST | +2023-05-10T11:17:13.326901Z | cluster1-0 joined | +2023-05-10T11:17:13.327013Z | 10.16.26.202 left | +2023-05-10T11:17:13.329192Z | 10.16.26.202 left | +2023-05-10T11:17:13.329351Z | PRIMARY(n=2) | +2023-05-10T11:21:49.944419Z | got SST from cluster1-0 | +2023-05-10T11:25:02.889253Z | preparing SST backup | +2023-05-10T11:25:38.882276Z | wsrep recovery | +2023-05-10T11:25:38.892098Z | JOINER -> JOINED | +2023-05-10T11:25:38.892510Z | JOINED -> SYNCED | +2023-05-10T11:26:23.849947Z | cluster1-0 joined | +2023-05-10T11:26:23.850027Z | cluster1-2 joined | +2023-05-10T11:26:23.851581Z | PRIMARY(n=3) | +2023-05-10T11:26:25.061573Z | local node will resync cluster1-2 | +2023-05-10T11:26:25.061656Z | SYNCED -> DONOR | +2023-05-10T11:26:25.078648Z | IST to cluster1-2(seqno:158423) | +2023-05-10T11:26:25.503200Z | IST will be used | +2023-05-10T11:26:26.522634Z | finished sending IST to cluster1-2 | +2023-05-10T11:26:26.522704Z | DESYNCED -> JOINED | +2023-05-10T11:26:26.523402Z | JOINED -> SYNCED | +2023-05-10T11:38:13.551353Z | cluster1-0 joined | +2023-05-10T11:38:13.551487Z | cluster1-2 left | +2023-05-10T11:38:13.553356Z | cluster1-2 left | +2023-05-10T11:38:13.553425Z | PRIMARY(n=2) | +2023-05-10T11:38:45.653905Z | cluster1-0 joined | +2023-05-10T11:38:45.653982Z | cluster1-2 joined | +2023-05-10T11:38:45.655569Z | PRIMARY(n=3) | +2023-05-10T11:38:46.845794Z | local node will resync cluster1-2 | +2023-05-10T11:38:46.845896Z | SYNCED -> DONOR | +2023-05-10T11:38:46.858192Z | IST to cluster1-2(seqno:158425) | +2023-05-10T11:38:47.296230Z | IST will be used | +2023-05-10T11:38:48.315759Z | finished sending IST to cluster1-2 | +2023-05-10T11:38:48.315831Z | DESYNCED -> JOINED | +2023-05-10T11:38:48.316431Z | JOINED -> SYNCED | +2023-05-10T11:42:43.168023Z | received shutdown | +2023-05-10T11:42:53.170702Z | NON-PRIMARY(n=1) | +2023-05-10T11:42:53.170909Z | SYNCED -> OPEN | +2023-05-10T11:42:53.171012Z | OPEN -> CLOSED | +2023-05-10T11:43:00.138274Z | shutdown complete | +2023-05-10T11:43:19.833967Z | starting(8.0.31) | +2023-05-10T11:43:19.836876Z | started(cluster) | +2023-05-10T11:43:19.838004Z | recovering gcache | +2023-05-10T11:43:19.838234Z | recovering gcache | + 10.16.27.93 + (node ip) + V + 10.16.27.67 +2023-05-10T11:43:20.356002Z | cluster1-0 joined | +2023-05-10T11:43:20.356029Z | cluster1-2 joined | +2023-05-10T11:43:20.855036Z | CLOSED -> OPEN | +2023-05-10T11:43:20.855274Z | PRIMARY(n=3) | +2023-05-10T11:43:20.856511Z | OPEN -> PRIMARY | +2023-05-10T11:43:21.550734Z | will receive IST(seqno:158427) | +2023-05-10T11:43:21.551987Z | cluster1-2 will resync local node | +2023-05-10T11:43:21.552057Z | PRIMARY -> JOINER | +2023-05-10T11:43:23.035052Z | got IST from cluster1-2 | +2023-05-10T11:43:28.800847Z | wsrep recovery | +2023-05-10T11:43:28.813599Z | IST received(seqno:158427) | +2023-05-10T11:43:28.814872Z | JOINER -> JOINED | +2023-05-10T11:43:28.815566Z | JOINED -> SYNCED | +2023-05-10T11:44:38.663126Z | cluster1-0 joined | +2023-05-10T11:44:38.663269Z | cluster1-2 left | +2023-05-10T11:44:38.665063Z | cluster1-2 left | +2023-05-10T11:44:38.665206Z | PRIMARY(n=2) | +2023-05-10T11:45:09.238513Z | cluster1-2 joined | +2023-05-10T11:45:09.238600Z | cluster1-0 joined | +2023-05-10T11:45:09.240227Z | PRIMARY(n=3) | +2023-05-10T11:45:10.423899Z | local node will resync cluster1-2 | +2023-05-10T11:45:10.423967Z | SYNCED -> DONOR | +2023-05-10T11:45:10.438294Z | IST to cluster1-2(seqno:158429) | +2023-05-10T11:45:10.874534Z | IST will be used | +2023-05-10T11:45:11.893200Z | finished sending IST to cluster1-2 | +2023-05-10T11:45:11.893227Z | DESYNCED -> JOINED | +2023-05-10T11:45:11.893759Z | JOINED -> SYNCED | +2023-05-10T11:50:01.194159Z | cluster1-0 joined | +2023-05-10T11:50:01.194272Z | cluster1-2 left | +2023-05-10T11:50:01.196426Z | cluster1-2 left | +2023-05-10T11:50:01.196471Z | PRIMARY(n=2) | +2023-05-10T11:50:33.633947Z | cluster1-0 joined | +2023-05-10T11:50:33.634035Z | cluster1-2 joined | +2023-05-10T11:50:33.635507Z | PRIMARY(n=3) | +2023-05-10T11:50:34.853691Z | local node will resync cluster1-2 | +2023-05-10T11:50:34.853767Z | SYNCED -> DONOR | +2023-05-10T11:50:34.867076Z | IST to cluster1-2(seqno:158431) | +2023-05-10T11:50:35.286248Z | IST will be used | +2023-05-10T11:50:36.307974Z | finished sending IST to cluster1-2 | +2023-05-10T11:50:36.308059Z | DESYNCED -> JOINED | +2023-05-10T11:50:36.308683Z | JOINED -> SYNCED | +2023-05-10T11:51:03.499185Z | cluster1-2 joined | +2023-05-10T11:51:03.499279Z | cluster1-0 left | +2023-05-10T11:51:03.500648Z | cluster1-0 left | +2023-05-10T11:51:03.500731Z | PRIMARY(n=2) | +2023-05-10T11:51:16.356400Z | received shutdown | +2023-05-10T11:51:27.360309Z | NON-PRIMARY(n=1) | +2023-05-10T11:51:27.360500Z | SYNCED -> OPEN | +2023-05-10T11:51:27.360599Z | OPEN -> CLOSED | +2023-05-10T11:51:34.025938Z | shutdown complete | +2023-05-10T11:51:58.584894Z | starting(8.0.31) | +2023-05-10T11:51:58.587873Z | started(cluster) | +2023-05-10T11:51:58.589002Z | recovering gcache | +2023-05-10T11:51:58.589835Z | recovering gcache | + 10.16.27.67 + (node ip) + V + 10.16.27.98 +2023-05-10T11:51:59.105411Z | cluster1-0 joined | +2023-05-10T11:51:59.105437Z | cluster1-2 joined | +2023-05-10T11:51:59.604194Z | CLOSED -> OPEN | +2023-05-10T11:51:59.604435Z | PRIMARY(n=3) | +2023-05-10T11:51:59.605624Z | OPEN -> PRIMARY | +2023-05-10T11:52:00.261139Z | will receive IST(seqno:158435) | +2023-05-10T11:52:00.262343Z | cluster1-2 will resync local node | +2023-05-10T11:52:00.262369Z | PRIMARY -> JOINER | +2023-05-10T11:52:01.769295Z | got IST from cluster1-2 | +2023-05-10T11:52:07.506267Z | wsrep recovery | +2023-05-10T11:52:07.520346Z | IST received(seqno:158435) | +2023-05-10T11:52:07.521397Z | JOINER -> JOINED | +2023-05-10T11:52:07.521904Z | JOINED -> SYNCED | +2023-05-10T12:56:32.224392Z | cluster1-0 joined | +2023-05-10T12:56:32.224466Z | garb joined | +2023-05-10T12:56:32.224483Z | cluster1-2 joined | +2023-05-10T12:56:32.225753Z | PRIMARY(n=4) | +2023-05-10T12:56:32.724770Z | cluster1-2 will resync garb | +2023-05-10T13:35:49.260734Z | cluster1-2 synced garb | +2023-05-10T13:35:51.219600Z | cluster1-0 joined | +2023-05-10T13:35:51.219621Z | cluster1-2 joined | +2023-05-10T13:35:51.219631Z | garb left | +2023-05-10T13:35:51.222978Z | garb left | +2023-05-10T13:35:51.223173Z | PRIMARY(n=3) | +2023-05-10T13:53:31.446329Z | cluster1-2 joined | +2023-05-10T13:53:31.446452Z | cluster1-0 left | +2023-05-10T13:53:31.447668Z | cluster1-0 left | +2023-05-10T13:53:31.447741Z | PRIMARY(n=2) | +2023-05-10T13:55:45.385943Z | cluster1-0 joined | +2023-05-10T13:55:45.386029Z | cluster1-2 joined | +2023-05-10T13:55:45.387576Z | PRIMARY(n=3) | +2023-05-10T13:55:46.518745Z | local node will resync cluster1-0 | +2023-05-10T13:55:46.518814Z | SYNCED -> DONOR | +2023-05-10T13:55:46.524750Z | IST to cluster1-0(seqno:158442) | +2023-05-10T13:55:57.967207Z | SST to cluster1-0 | +2023-05-10T14:29:12.924344Z | finished sending IST to cluster1-0 | +2023-05-10T14:29:12.924417Z | DESYNCED -> JOINED | +2023-05-10T14:29:12.925049Z | JOINED -> SYNCED | +2023-05-11T02:00:39.127163Z | cluster1-0 joined | +2023-05-11T02:00:39.127261Z | garb joined | +2023-05-11T02:00:39.127285Z | cluster1-2 joined | +2023-05-11T02:00:39.128891Z | PRIMARY(n=4) | +2023-05-11T02:00:39.628105Z | cluster1-2 will resync garb | +2023-05-11T02:46:06.517307Z | cluster1-2 synced garb | +2023-05-11T02:46:08.245842Z | cluster1-0 joined | +2023-05-11T02:46:08.245883Z | cluster1-2 joined | +2023-05-11T02:46:08.245906Z | garb left | +2023-05-11T02:46:08.246895Z | garb left | +2023-05-11T02:46:08.246963Z | PRIMARY(n=3) | +2023-05-12T02:00:37.685744Z | cluster1-0 joined | +2023-05-12T02:00:37.685839Z | garb joined | +2023-05-12T02:00:37.685882Z | cluster1-2 joined | +2023-05-12T02:00:37.687768Z | PRIMARY(n=4) | +2023-05-12T02:00:38.186730Z | cluster1-2 will resync garb | +2023-05-12T08:55:58.040880Z | garb suspected to be down | +2023-05-12T08:55:59.042955Z | cluster1-0 joined | +2023-05-12T08:55:59.043017Z | cluster1-2 joined | +2023-05-12T08:55:59.044612Z | garb left | +2023-05-12T08:55:59.044648Z | PRIMARY(n=3) | +2023-05-12T08:56:23.810378Z | cluster1-0 joined | +2023-05-12T08:56:23.810472Z | cluster1-2 joined | +2023-05-12T08:56:23.810496Z | garb joined | +2023-05-12T08:56:23.811966Z | PRIMARY(n=4) | +2023-05-12T08:56:24.311102Z | local node will resync garb | +2023-05-12T08:56:24.311127Z | SYNCED -> DONOR | +2023-05-12T08:56:35.819395Z | SST to garb | +2023-05-12T09:42:29.055858Z | finished sending SST to garb | +2023-05-12T09:42:29.055912Z | DESYNCED -> JOINED | +2023-05-12T09:42:29.056654Z | JOINED -> SYNCED | +2023-05-12T09:42:31.373457Z | cluster1-0 joined | +2023-05-12T09:42:31.373480Z | cluster1-2 joined | +2023-05-12T09:42:31.373494Z | garb left | +2023-05-12T09:42:31.374839Z | garb left | +2023-05-12T09:42:31.374912Z | PRIMARY(n=3) | +2023-05-12T18:59:18.005493Z | received shutdown | +2023-05-12T19:13:56.275184Z | starting(8.0.31) | +2023-05-12T19:13:56.279645Z | started(cluster) | +2023-05-12T19:13:56.280946Z | recovering gcache | +2023-05-12T19:13:56.761747Z | recovering gcache | +2023-05-12T19:13:56.881098Z | cluster1-0 joined | +2023-05-12T19:13:56.881130Z | cluster1-2 joined | +2023-05-12T19:13:57.323577Z | CLOSED -> OPEN | +2023-05-12T19:13:57.323870Z | PRIMARY(n=3) | +2023-05-12T19:13:57.325487Z | OPEN -> PRIMARY | +2023-05-12T19:13:57.953877Z | will receive IST(seqno:2584375) | +2023-05-12T19:13:57.954776Z | cluster1-0 will resync local node | +2023-05-12T19:13:57.954842Z | PRIMARY -> JOINER | +2023-05-12T19:13:59.540001Z | got IST from cluster1-0 | +2023-05-12T19:14:05.791982Z | wsrep recovery | +2023-05-12T19:14:07.444245Z | IST received(seqno:2584375) | +2023-05-12T19:14:07.445205Z | JOINER -> JOINED | +2023-05-12T19:14:07.445797Z | JOINED -> SYNCED | +2023-05-12T19:18:57.960770Z | received shutdown | +2023-05-12T19:29:33.473713Z | starting(8.0.31) | +2023-05-12T19:29:33.477931Z | started(cluster) | +2023-05-12T19:29:33.479273Z | recovering gcache | +2023-05-12T19:29:34.017562Z | recovering gcache | +2023-05-12T19:29:34.105777Z | cluster1-0 joined | +2023-05-12T19:29:34.105800Z | cluster1-2 joined | +2023-05-12T19:29:34.597464Z | CLOSED -> OPEN | +2023-05-12T19:29:34.597650Z | PRIMARY(n=3) | +2023-05-12T19:29:34.599141Z | OPEN -> PRIMARY | +2023-05-12T19:29:35.197125Z | will receive IST(seqno:2586360) | +2023-05-12T19:29:35.198117Z | cluster1-0 will resync local node | +2023-05-12T19:29:35.198184Z | PRIMARY -> JOINER | +2023-05-12T19:29:37.606722Z | got IST from cluster1-0 | +2023-05-12T19:29:44.510160Z | wsrep recovery | +2023-05-12T19:29:45.181433Z | IST received(seqno:2586360) | +2023-05-12T19:29:45.181948Z | JOINER -> JOINED | +2023-05-12T19:29:45.182426Z | JOINED -> SYNCED | +2023-05-13T02:00:39.051996Z | garb joined | +2023-05-13T02:00:39.052078Z | cluster1-0 joined | +2023-05-13T02:00:39.052095Z | cluster1-2 joined | +2023-05-13T02:00:39.053275Z | PRIMARY(n=4) | +2023-05-13T02:00:39.489721Z | local node will resync garb | +2023-05-13T02:00:39.489739Z | SYNCED -> DONOR | +2023-05-13T02:00:51.007793Z | SST to garb | +2023-05-13T02:45:57.327463Z | finished sending SST to garb | +2023-05-13T02:45:57.327542Z | DESYNCED -> JOINED | +2023-05-13T02:45:57.328249Z | JOINED -> SYNCED | +2023-05-13T02:45:59.525579Z | cluster1-0 joined | +2023-05-13T02:45:59.525600Z | cluster1-2 joined | +2023-05-13T02:45:59.525612Z | garb left | +2023-05-13T02:45:59.526936Z | garb left | +2023-05-13T02:45:59.527009Z | PRIMARY(n=3) | +2023-05-13T06:54:00.143632Z | cluster1-2 joined | +2023-05-13T06:54:00.143727Z | cluster1-0 left | +2023-05-13T06:54:00.144901Z | cluster1-0 left | +2023-05-13T06:54:00.144944Z | PRIMARY(n=2) | +2023-05-13T06:55:57.140277Z | (repeated x2275)too many connections | +2023-05-13T06:56:15.493714Z | too many connections | +2023-05-18T14:45:40.446083Z | starting(8.0.31)(post.processing.log) | +2023-05-18T14:45:47.537181Z | started(standalone)(post.processing.log) | +2023-05-13T06:56:15.501414Z | (repeated x3964)too many connections | +2023-05-13T06:56:45.240370Z | too many connections | +2023-05-13T06:57:31.547392Z | cluster1-0 joined | +2023-05-13T06:57:31.547466Z | cluster1-2 joined | +2023-05-13T06:57:31.548940Z | PRIMARY(n=3) | +2023-05-13T06:57:32.679563Z | local node will resync cluster1-0 | +2023-05-13T06:57:32.679642Z | SYNCED -> DONOR | +2023-05-13T06:57:32.693189Z | IST to cluster1-0(seqno:2987828) | +2023-05-13T06:57:33.097517Z | IST will be used | +2023-05-13T06:57:59.222707Z | finished sending IST to cluster1-0 | +2023-05-13T06:57:59.223896Z | DESYNCED -> JOINED | +2023-05-13T06:57:59.224286Z | JOINED -> SYNCED | +2023-05-14T02:00:39.346372Z | garb joined | +2023-05-14T02:00:39.346464Z | cluster1-0 joined | +2023-05-14T02:00:39.346489Z | cluster1-2 joined | +2023-05-14T02:00:39.348244Z | PRIMARY(n=4) | +2023-05-14T02:00:39.847436Z | local node will resync garb | +2023-05-14T02:00:39.847459Z | SYNCED -> DONOR | +2023-05-14T02:00:51.320665Z | SST to garb | +2023-05-14T02:45:02.456243Z | finished sending SST to garb | +2023-05-14T02:45:02.456314Z | DESYNCED -> JOINED | +2023-05-14T02:45:02.457052Z | JOINED -> SYNCED | +2023-05-14T02:45:04.774986Z | cluster1-0 joined | +2023-05-14T02:45:04.775082Z | cluster1-2 joined | +2023-05-14T02:45:04.775109Z | garb left | +2023-05-14T02:45:04.776467Z | garb left | +2023-05-14T02:45:04.776542Z | PRIMARY(n=3) | +2023-05-15T02:00:37.325176Z | garb joined | +2023-05-15T02:00:37.325301Z | cluster1-0 joined | +2023-05-15T02:00:37.325330Z | cluster1-2 joined | +2023-05-15T02:00:37.327052Z | PRIMARY(n=4) | +2023-05-15T02:00:37.826406Z | local node will resync garb | +2023-05-15T02:00:37.826427Z | SYNCED -> DONOR | +2023-05-15T02:00:49.304365Z | SST to garb | +2023-05-15T02:43:52.459283Z | finished sending SST to garb | +2023-05-15T02:43:52.459313Z | DESYNCED -> JOINED | +2023-05-15T02:43:52.459898Z | JOINED -> SYNCED | +2023-05-15T02:43:54.313680Z | cluster1-0 joined | +2023-05-15T02:43:54.313721Z | cluster1-2 joined | +2023-05-15T02:43:54.313732Z | garb left | +2023-05-15T02:43:55.315909Z | garb left | +2023-05-15T02:43:55.315972Z | PRIMARY(n=3) | +2023-05-15T05:00:37.943518Z | cluster1-0 joined | +2023-05-15T05:00:37.943607Z | garb joined | +2023-05-15T05:00:37.943631Z | cluster1-2 joined | +2023-05-15T05:00:37.945220Z | PRIMARY(n=4) | +2023-05-15T05:00:38.387001Z | local node will resync garb | +2023-05-15T05:00:38.387063Z | SYNCED -> DONOR | +2023-05-15T05:00:49.866151Z | SST to garb | +2023-05-15T05:44:06.033520Z | finished sending SST to garb | +2023-05-15T05:44:06.033594Z | DESYNCED -> JOINED | +2023-05-15T05:44:06.034340Z | JOINED -> SYNCED | +2023-05-15T05:44:09.112339Z | cluster1-0 joined | +2023-05-15T05:44:09.112422Z | cluster1-2 joined | +2023-05-15T05:44:09.112451Z | garb left | +2023-05-15T05:44:09.114037Z | garb left | +2023-05-15T05:44:09.114066Z | PRIMARY(n=3) | +2023-05-15T17:18:09.283033Z | cluster1-2 joined | +2023-05-15T17:18:09.283176Z | cluster1-0 left | +2023-05-15T17:18:09.284393Z | cluster1-0 left | +2023-05-15T17:18:09.284417Z | PRIMARY(n=2) | +2023-05-15T17:19:28.022611Z | cluster1-0 joined | +2023-05-15T17:19:28.022710Z | cluster1-2 joined | +2023-05-15T17:19:28.023947Z | PRIMARY(n=3) | +2023-05-15T17:19:29.187799Z | local node will resync cluster1-0 | +2023-05-15T17:19:29.187866Z | SYNCED -> DONOR | +2023-05-15T17:19:29.201328Z | IST to cluster1-0(seqno:5408371) | +2023-05-15T17:19:29.651209Z | IST will be used | +2023-05-15T17:19:30.672304Z | finished sending IST to cluster1-0 | +2023-05-15T17:19:30.672382Z | DESYNCED -> JOINED | +2023-05-15T17:19:30.672874Z | JOINED -> SYNCED | +2023-05-16T02:00:39.155225Z | garb joined | +2023-05-16T02:00:39.155270Z | cluster1-0 joined | +2023-05-16T02:00:39.155278Z | cluster1-2 joined | +2023-05-16T02:00:39.156563Z | PRIMARY(n=4) | +2023-05-16T02:00:39.393865Z | local node will resync garb | +2023-05-16T02:00:39.393887Z | SYNCED -> DONOR | +2023-05-16T02:00:50.852843Z | SST to garb | +2023-05-16T02:26:01.150948Z | garb joined | +2023-05-16T02:26:01.151019Z | cluster1-2 joined | +2023-05-16T02:26:01.151037Z | cluster1-0 left | +2023-05-16T02:26:01.152236Z | cluster1-0 left | +2023-05-16T02:26:01.152322Z | PRIMARY(n=3) | +2023-05-16T02:41:51.091600Z | finished sending SST to garb | +2023-05-16T02:41:51.091676Z | DESYNCED -> JOINED | +2023-05-16T02:41:51.092224Z | JOINED -> SYNCED | +2023-05-16T02:41:53.415858Z | cluster1-2 joined | +2023-05-16T02:41:53.416002Z | garb left | +2023-05-16T02:41:53.417044Z | garb left | +2023-05-16T02:41:53.417130Z | PRIMARY(n=2) | +2023-05-16T02:42:36.811151Z | cluster1-0 joined | +2023-05-16T02:42:36.811232Z | cluster1-2 joined | +2023-05-16T02:42:36.812646Z | PRIMARY(n=3) | +2023-05-16T02:42:37.922175Z | local node will resync cluster1-0 | +2023-05-16T02:42:37.922257Z | SYNCED -> DONOR | +2023-05-16T02:42:37.935365Z | IST to cluster1-0(seqno:5697478) | +2023-05-16T02:42:38.416273Z | IST will be used | +2023-05-16T02:42:39.440136Z | finished sending IST to cluster1-0 | +2023-05-16T02:42:39.440201Z | DESYNCED -> JOINED | +2023-05-16T02:42:39.440867Z | JOINED -> SYNCED | +2023-05-16T02:45:10.902553Z | (repeated x105)too many connections | +2023-05-16T02:45:14.464524Z | too many connections | +2023-05-16T02:46:18.007412Z | received shutdown | +2023-05-16T02:56:57.328220Z | starting(8.0.31) | +2023-05-16T02:56:57.339616Z | started(cluster) | +2023-05-16T02:56:57.340972Z | recovering gcache | +2023-05-16T02:56:57.994563Z | recovering gcache | +2023-05-16T02:56:58.598150Z | cluster1-0 joined | +2023-05-16T02:56:58.598176Z | cluster1-2 joined | +2023-05-16T02:56:59.096957Z | CLOSED -> OPEN | +2023-05-16T02:56:59.097146Z | PRIMARY(n=3) | +2023-05-16T02:56:59.098728Z | OPEN -> PRIMARY | +2023-05-16T02:56:59.754093Z | will receive IST(seqno:5698287) | +2023-05-16T02:56:59.755433Z | cluster1-0 will resync local node | +2023-05-16T02:56:59.755465Z | PRIMARY -> JOINER | +2023-05-16T02:56:59.757859Z | cluster1-0 failed to sync cluster1-1 | +2023-05-16T02:56:59.757877Z | will never receive SST, aborting | +2023-05-16T02:56:59.758599Z | terminated | +2023-05-16T02:56:59.758607Z | former SST cancelled | +2023-05-16T02:56:59Z | crash: got signal 11 | +2023-05-16T02:58:05.418798Z | starting(8.0.31) | +2023-05-16T02:58:05.425961Z | started(cluster) | +2023-05-16T02:58:05.427232Z | recovering gcache | +2023-05-16T02:58:05.824106Z | recovering gcache | +2023-05-16T02:58:06.377247Z | cluster1-0 joined | +2023-05-16T02:58:06.377275Z | cluster1-2 joined | +2023-05-16T02:58:06.876188Z | CLOSED -> OPEN | +2023-05-16T02:58:06.876404Z | PRIMARY(n=3) | +2023-05-16T02:58:06.877762Z | OPEN -> PRIMARY | +2023-05-16T02:58:07.513826Z | will receive SST | +2023-05-16T02:58:07.514763Z | cluster1-0 will resync local node | +2023-05-16T02:58:07.514835Z | PRIMARY -> JOINER | +2023-05-16T02:58:07.522280Z | cluster1-0 failed to sync cluster1-1 | +2023-05-16T02:58:07.522334Z | will never receive SST, aborting | +2023-05-16T02:58:07.523605Z | terminated | +2023-05-16T02:58:07.523621Z | former SST cancelled | +2023-05-16T02:58:07Z | crash: got signal 11 | +2023-05-16T02:59:15.251276Z | starting(8.0.31) | +2023-05-16T02:59:15.257433Z | started(cluster) | +2023-05-16T02:59:15.258668Z | recovering gcache | +2023-05-16T02:59:15.688672Z | recovering gcache | +2023-05-16T02:59:44.719012Z | cluster1-0 joined | +2023-05-16T02:59:44.719042Z | cluster1-2 joined | +2023-05-16T02:59:45.218169Z | CLOSED -> OPEN | +2023-05-16T02:59:45.218407Z | PRIMARY(n=3) | +2023-05-16T02:59:45.219528Z | OPEN -> PRIMARY | +2023-05-16T02:59:45.884309Z | will receive SST | +2023-05-16T02:59:45.885230Z | cluster1-0 will resync local node | +2023-05-16T02:59:45.885308Z | PRIMARY -> JOINER | +2023-05-16T02:59:45.893113Z | cluster1-0 failed to sync cluster1-1 | +2023-05-16T02:59:45.893164Z | will never receive SST, aborting | +2023-05-16T02:59:45.894389Z | terminated | +2023-05-16T02:59:45.894407Z | former SST cancelled | +2023-05-16T02:59:45Z | crash: got signal 11 | +2023-05-16T03:00:01.228500Z | SST error | +2023-05-16T03:01:22.029475Z | starting(8.0.31) | +2023-05-16T03:01:22.036060Z | started(cluster) | +2023-05-16T03:01:22.037309Z | (repeated x1)recovering gcache | +2023-05-16T03:01:33.094706Z | recovering gcache | +2023-05-16T03:01:59.670468Z | cluster1-0 joined | +2023-05-16T03:01:59.670482Z | cluster1-2 joined | +2023-05-16T03:02:00.169304Z | CLOSED -> OPEN | +2023-05-16T03:02:00.169482Z | PRIMARY(n=3) | +2023-05-16T03:02:00.171272Z | OPEN -> PRIMARY | +2023-05-16T03:02:00.814180Z | will receive SST | +2023-05-16T03:02:00.815258Z | cluster1-0 will resync local node | +2023-05-16T03:02:00.815327Z | PRIMARY -> JOINER | +2023-05-16T03:02:00.855489Z | cluster1-0 failed to sync cluster1-1 | +2023-05-16T03:02:00.855539Z | will never receive SST, aborting | +2023-05-16T03:02:00.857051Z | terminated | +2023-05-16T03:02:00.857070Z | former SST cancelled | +2023-05-16T03:02:00Z | crash: got signal 11 | +2023-05-16T03:04:16.906158Z | starting(8.0.31) | +2023-05-16T03:04:16.912054Z | started(cluster) | +2023-05-16T03:04:16.913132Z | (repeated x1)recovering gcache | +2023-05-16T03:04:27.424753Z | recovering gcache | +2023-05-16T03:04:51.780849Z | cluster1-2 joined | +2023-05-16T03:04:52.279974Z | CLOSED -> OPEN | +2023-05-16T03:04:52.280182Z | PRIMARY(n=2) | +2023-05-16T03:04:52.281364Z | OPEN -> PRIMARY | +2023-05-16T03:04:52.954535Z | will receive SST | +2023-05-16T03:04:52.955179Z | cannot find donor | +2023-05-16T03:04:53.956078Z | cannot find donor | +2023-05-16T03:04:54.956958Z | (repeated x97)cannot find donor | +2023-05-16T03:06:33.039898Z | cannot find donor | +2023-05-16T03:06:34.023406Z | timeout from donor in gtid/keyring stage | +2023-05-16T03:06:34.030283Z | SST error | +2023-05-16T03:06:34.040289Z | terminated | +2023-05-16T03:06:34.040309Z | former SST cancelled | +2023-05-16T03:06:34Z | crash: got signal 11 | +2023-05-16T07:51:48.142342Z | starting(8.0.31) | +2023-05-16T07:51:48.148592Z | started(cluster) | +2023-05-16T07:51:48.149740Z | (repeated x1)recovering gcache | +2023-05-16T07:51:58.379294Z | recovering gcache | +2023-05-16T07:52:26.431252Z | CLOSED -> OPEN | +2023-05-16T07:52:26.431427Z | NON-PRIMARY(n=1) | +2023-05-16T10:03:18.655740Z starting(8.0.31) | | +2023-05-16T10:03:18.658268Z started(cluster) | | +2023-05-16T10:03:18.659239Z no grastate.dat file | | +2023-05-16T10:03:18.659248Z bootstrapping(empty grastate) | | +2023-05-16T10:03:18.659274Z safe_to_bootstrap: 1 | | +2023-05-16T10:03:19.700941Z cluster1-2 joined | | +2023-05-16T10:03:19.700942Z | cluster1-2 joined | +2023-05-16T10:03:19.700950Z cluster1-1 joined | | +2023-05-16T10:03:19.701041Z | cluster1-0 joined | +2023-05-16T10:03:19.702446Z | PRIMARY(n=3) | +2023-05-16T10:03:20.174154Z CLOSED -> OPEN | | +2023-05-16T10:03:20.174415Z PRIMARY(n=3) | | +2023-05-16T10:03:20.175471Z | OPEN -> PRIMARY | +2023-05-16T10:03:20.175651Z OPEN -> PRIMARY | | +2023-05-16T10:03:20.825855Z | will receive SST | +2023-05-16T10:03:20.826785Z cluster1-2 will resync cluster1-1 | | +2023-05-16T10:03:20.826808Z | cluster1-2 will resync local node | +2023-05-16T10:03:20.826874Z | PRIMARY -> JOINER | +2023-05-16T10:03:20.829695Z will receive SST | | +2023-05-16T10:03:20.830129Z | cluster1-0 cannot find donor | +2023-05-16T10:03:20.830169Z cluster1-0 cannot find donor | | +2023-05-16T10:03:21.636432Z | receiving SST | +2023-05-16T10:03:21.831094Z | (repeated x98)cluster1-0 cannot find donor | +2023-05-16T10:03:21.831193Z cluster1-0 cannot find donor | | +2023-05-16T10:03:22.832160Z (repeated x97)cluster1-0 cannot find donor | | +2023-05-16T10:05:00.937424Z | cluster1-0 cannot find donor | +2023-05-16T10:05:00.937441Z cluster1-0 cannot find donor | | +2023-05-16T10:05:01.909476Z timeout from donor in gtid/keyring stage | | +2023-05-16T10:05:01.916956Z SST error | | +2023-05-16T10:05:01.918653Z | cluster1-2 joined | +2023-05-16T10:05:01.918732Z | cluster1-0 left | +2023-05-16T10:05:01.918737Z NON-PRIMARY(n=1) | | +2023-05-16T10:05:01.918924Z PRIMARY -> OPEN | | +2023-05-16T10:05:01.919021Z OPEN -> CLOSED | | +2023-05-16T10:05:01.919173Z terminated | | +2023-05-16T10:05:01.919181Z former SST cancelled | | +2023-05-16T10:05:01Z crash: got signal 11 | | +2023-05-16T10:05:01.920148Z | cluster1-0 left | +2023-05-16T10:05:01.920217Z | PRIMARY(n=2) | +2023-05-16T10:47:53.548849Z | got SST from cluster1-2 | +2023-05-16T10:47:53.581506Z | preparing SST backup | +2023-05-16T10:48:33.507965Z | wsrep recovery | +2023-05-16T10:48:33.533202Z | JOINER -> JOINED | +2023-05-16T10:48:34.865824Z | JOINED -> SYNCED | +2023-05-16T10:48:46.119407Z starting(8.0.31) | | +2023-05-16T10:48:46.121988Z started(cluster) | | +2023-05-16T10:48:46.122926Z safe_to_bootstrap: 1 | | +2023-05-16T10:48:46.636641Z | cluster1-2 joined | +2023-05-16T10:48:46.636729Z | cluster1-0 joined | +2023-05-16T10:48:46.636865Z cluster1-2 joined | | +2023-05-16T10:48:46.636892Z cluster1-1 joined | | +2023-05-16T10:48:46.638432Z | PRIMARY(n=3) | +2023-05-16T10:48:47.136082Z CLOSED -> OPEN | | +2023-05-16T10:48:47.136335Z PRIMARY(n=3) | | +2023-05-16T10:48:47.137440Z OPEN -> PRIMARY | | +2023-05-16T10:48:47.813851Z will receive SST | | +2023-05-16T10:48:47.814607Z | cluster1-2 will resync cluster1-0 | +2023-05-16T10:48:47.814777Z cluster1-2 will resync local node | | +2023-05-16T10:48:47.814846Z PRIMARY -> JOINER | | +2023-05-16T10:48:48.623834Z receiving SST | | +2023-05-16T11:32:58.984427Z | cluster1-2 synced cluster1-0 | +2023-05-16T11:32:58.984956Z got SST from cluster1-2 | | +2023-05-16T11:32:59.020381Z preparing SST backup | | +2023-05-16T11:33:35.431876Z wsrep recovery | | +2023-05-16T11:33:35.443968Z JOINER -> JOINED | | +2023-05-16T11:33:36.623324Z JOINED -> SYNCED | | +2023-05-16T12:28:44.905840Z | cluster1-2 suspected to be down | +2023-05-16T12:28:45.406765Z cluster1-2 suspected to be down | | +2023-05-16T12:28:45.907278Z | cluster1-0 joined | +2023-05-16T12:28:45.907408Z cluster1-1 joined | | +2023-05-16T12:28:45.908701Z cluster1-2 left | | +2023-05-16T12:28:45.908766Z PRIMARY(n=2) | | +2023-05-16T12:28:45.909369Z | cluster1-2 left | +2023-05-16T12:28:45.909445Z | PRIMARY(n=2) | +2023-05-16T12:29:33.780957Z | cluster1-2 joined | +2023-05-16T12:29:33.781003Z | cluster1-0 joined | +2023-05-16T12:29:33.781162Z cluster1-2 joined | | +2023-05-16T12:29:33.781197Z cluster1-1 joined | | +2023-05-16T12:29:33.781849Z | PRIMARY(n=3) | +2023-05-16T12:29:33.782097Z PRIMARY(n=3) | | +2023-05-16T12:29:34.807852Z | local node will resync cluster1-2 | +2023-05-16T12:29:34.807945Z | SYNCED -> DONOR | +2023-05-16T12:29:34.808108Z cluster1-1 will resync cluster1-2 | | +2023-05-16T12:29:34.826520Z | IST to cluster1-2(seqno:90134) | +2023-05-16T12:29:35.291536Z | IST will be used | +2023-05-16T12:29:36.348959Z | finished sending IST to cluster1-2 | +2023-05-16T12:29:36.349031Z | DESYNCED -> JOINED | +2023-05-16T12:29:36.349099Z cluster1-1 synced cluster1-2 | | +2023-05-16T12:29:36.349715Z | JOINED -> SYNCED | +2023-05-17T02:00:38.559825Z cluster1-2 joined | | +2023-05-17T02:00:38.559903Z | cluster1-2 joined | +2023-05-17T02:00:38.559921Z cluster1-1 joined | | +2023-05-17T02:00:38.559947Z garb joined | | +2023-05-17T02:00:38.559998Z | cluster1-0 joined | +2023-05-17T02:00:38.560023Z | garb joined | +2023-05-17T02:00:38.561718Z | PRIMARY(n=4) | +2023-05-17T02:00:38.561911Z PRIMARY(n=4) | | +2023-05-17T02:00:39.060626Z cluster1-2 will resync garb | | +2023-05-17T02:00:39.060663Z | cluster1-2 will resync garb | +2023-05-17T02:42:53.094651Z | cluster1-2 synced garb | +2023-05-17T02:42:53.094869Z cluster1-2 synced garb | | +2023-05-17T02:42:55.155501Z | cluster1-2 joined | +2023-05-17T02:42:55.155542Z | cluster1-0 joined | +2023-05-17T02:42:55.155565Z | garb left | +2023-05-17T02:42:55.155574Z cluster1-2 joined | | +2023-05-17T02:42:55.155588Z cluster1-1 joined | | +2023-05-17T02:42:55.155596Z garb left | | +2023-05-17T02:42:55.156821Z | garb left | +2023-05-17T02:42:55.156903Z | PRIMARY(n=3) | +2023-05-17T02:42:55.156912Z garb left | | +2023-05-17T02:42:55.156990Z PRIMARY(n=3) | | +2023-05-18T02:00:38.610721Z | cluster1-2 joined | +2023-05-18T02:00:38.610738Z cluster1-2 joined | | +2023-05-18T02:00:38.610778Z | cluster1-0 joined | +2023-05-18T02:00:38.610794Z | garb joined | +2023-05-18T02:00:38.610821Z cluster1-1 joined | | +2023-05-18T02:00:38.610846Z garb joined | | +2023-05-18T02:00:38.612270Z | PRIMARY(n=4) | +2023-05-18T02:00:38.612336Z PRIMARY(n=4) | | +2023-05-18T02:00:39.111354Z | cluster1-2 will resync garb | +2023-05-18T02:00:39.111362Z cluster1-2 will resync garb | | +2023-05-18T02:43:21.261195Z | cluster1-2 synced garb | +2023-05-18T02:43:21.261218Z cluster1-2 synced garb | | +2023-05-18T02:43:23.331315Z cluster1-2 joined | | +2023-05-18T02:43:23.331333Z | cluster1-2 joined | +2023-05-18T02:43:23.331355Z cluster1-1 joined | | +2023-05-18T02:43:23.331373Z | cluster1-0 joined | +2023-05-18T02:43:23.331414Z | garb left | +2023-05-18T02:43:23.331418Z garb left | | +2023-05-18T02:43:23.332680Z | garb left | +2023-05-18T02:43:23.332740Z | PRIMARY(n=3) | +2023-05-18T02:43:23.333044Z garb left | | +2023-05-18T02:43:23.333107Z PRIMARY(n=3) | | +2023-05-18T07:59:17.124743Z received shutdown | | +2023-05-18T07:59:17.968467Z | received shutdown | +2023-05-18T07:59:27.128081Z | cluster1-2 joined | +2023-05-18T07:59:27.128191Z NON-PRIMARY(n=1) | | +2023-05-18T07:59:27.128200Z | cluster1-0 left | +2023-05-18T07:59:27.128414Z SYNCED -> OPEN | | +2023-05-18T07:59:27.128520Z OPEN -> CLOSED | | +2023-05-18T07:59:27.129727Z | cluster1-0 left | +2023-05-18T07:59:27.129794Z | PRIMARY(n=2) | +2023-05-18T07:59:28.972265Z | NON-PRIMARY(n=1) | +2023-05-18T07:59:28.972428Z | SYNCED -> OPEN | +2023-05-18T07:59:28.972542Z | OPEN -> CLOSED | +2023-05-18T08:10:55.079377Z starting(8.0.31) | | +2023-05-18T08:10:55.082696Z started(cluster) | | +2023-05-18T08:10:55.083755Z safe_to_bootstrap: 1 | | +2023-05-18T08:10:55.084042Z recovering gcache | | +2023-05-18T08:10:56.027905Z recovering gcache | | +2023-05-18T08:10:56.234199Z bootstrapping | | +2023-05-18T08:10:56.235055Z CLOSED -> OPEN | | +2023-05-18T08:10:56.235277Z PRIMARY(n=1) | | +2023-05-18T08:10:56.235913Z (restored)OPEN -> JOINED | | +2023-05-18T08:10:56.235984Z JOINED -> SYNCED | | +2023-05-18T08:12:26.802815Z cluster1-2 joined | | +2023-05-18T08:12:26.804265Z PRIMARY(n=2) | | +2023-05-18T08:12:28.032556Z local node will resync cluster1-2 | | +2023-05-18T08:12:28.032650Z SYNCED -> DONOR | | +2023-05-18T08:12:28.038966Z IST to cluster1-2(seqno:173) | | +2023-05-18T08:12:39.503380Z SST to cluster1-2 | | +2023-05-18T08:50:05.834112Z finished sending IST to cluster1-2 | | +2023-05-18T08:50:05.834143Z DESYNCED -> JOINED | | +2023-05-18T08:50:05.834494Z JOINED -> SYNCED | | +2023-05-18T08:51:05.420478Z | starting(8.0.31) | +2023-05-18T08:51:05.429007Z | started(cluster) | +2023-05-18T08:51:05.430480Z | recovering gcache | +2023-05-18T08:51:06.624095Z | recovering gcache | +2023-05-18T08:51:07.210530Z cluster1-1 joined | | +2023-05-18T08:51:07.210570Z cluster1-2 joined | | +2023-05-18T08:51:07.211254Z | cluster1-0 joined | +2023-05-18T08:51:07.211282Z | cluster1-2 joined | +2023-05-18T08:51:07.211878Z PRIMARY(n=3) | | +2023-05-18T08:51:07.470375Z | CLOSED -> OPEN | +2023-05-18T08:51:07.470547Z | PRIMARY(n=3) | +2023-05-18T08:51:07.472017Z | OPEN -> PRIMARY | +2023-05-18T08:51:08.111952Z | will receive SST | +2023-05-18T08:51:08.112104Z local node will resync cluster1-1 | | +2023-05-18T08:51:08.112226Z SYNCED -> DONOR | | +2023-05-18T08:51:08.112703Z | cluster1-0 will resync local node | +2023-05-18T08:51:08.112779Z | PRIMARY -> JOINER | +2023-05-18T08:51:08.119652Z IST to cluster1-1(seqno:17135) | | +2023-05-18T08:51:08.932698Z | receiving SST | +2023-05-18T08:51:19.607014Z SST to cluster1-1 | | +2023-05-18T09:29:06.931404Z finished sending IST to cluster1-1 | | +2023-05-18T09:29:06.931474Z DESYNCED -> JOINED | | +2023-05-18T09:29:06.932023Z | got SST from cluster1-0 | +2023-05-18T09:29:06.932145Z JOINED -> SYNCED | | +2023-05-18T09:32:43.074179Z | preparing SST backup | +2023-05-18T09:33:24.024352Z | wsrep recovery | +2023-05-18T09:33:24.042895Z | JOINER -> JOINED | +2023-05-18T09:33:25.567760Z | JOINED -> SYNCED | +2023-05-18T11:04:37.966152Z | received shutdown | +2023-05-18T11:04:47.969182Z | NON-PRIMARY(n=1) | +2023-05-18T11:04:47.969361Z | SYNCED -> OPEN | +2023-05-18T11:04:47.969459Z | OPEN -> CLOSED | +2023-05-18T11:04:47.969461Z cluster1-2 joined | | +2023-05-18T11:04:47.969598Z cluster1-1 left | | +2023-05-18T11:04:47.970895Z cluster1-1 left | | +2023-05-18T11:04:47.971039Z PRIMARY(n=2) | | +2023-05-18T11:15:15.567539Z | starting(8.0.31) | +2023-05-18T11:15:15.575256Z | started(cluster) | +2023-05-18T11:15:15.576587Z | recovering gcache | +2023-05-18T11:15:16.671487Z | recovering gcache | +2023-05-18T11:15:17.018482Z | cluster1-0 joined | +2023-05-18T11:15:17.018507Z | cluster1-2 joined | +2023-05-18T11:15:17.018917Z cluster1-1 joined | | +2023-05-18T11:15:17.019843Z cluster1-2 joined | | +2023-05-18T11:15:17.020877Z PRIMARY(n=3) | | +2023-05-18T11:15:17.481649Z | CLOSED -> OPEN | +2023-05-18T11:15:17.481816Z | PRIMARY(n=3) | +2023-05-18T11:15:17.483605Z | OPEN -> PRIMARY | +2023-05-18T11:15:18.115949Z | will receive SST | +2023-05-18T11:15:18.116912Z | cluster1-0 will resync local node | +2023-05-18T11:15:18.116942Z | PRIMARY -> JOINER | +2023-05-18T11:15:18.117425Z local node will resync cluster1-1 | | +2023-05-18T11:15:18.117522Z SYNCED -> DONOR | | +2023-05-18T11:15:18.124298Z IST to cluster1-1(seqno:96339) | | +2023-05-18T11:15:18.940943Z | receiving SST | +2023-05-18T11:15:29.599110Z SST to cluster1-1 | | +2023-05-18T11:48:59.162092Z cluster1-1 joined | | +2023-05-18T11:48:59.162198Z | cluster1-0 joined | +2023-05-18T11:48:59.162368Z | cluster1-2 left | +2023-05-18T11:48:59.162370Z cluster1-2 left | | +2023-05-18T11:48:59.163992Z | cluster1-2 left | +2023-05-18T11:48:59.164065Z | PRIMARY(n=2) | +2023-05-18T11:48:59.164368Z cluster1-2 left | | +2023-05-18T11:48:59.164431Z PRIMARY(n=2) | | +2023-05-18T11:52:34.738198Z finished sending IST to cluster1-1 | | +2023-05-18T11:52:34.738230Z DESYNCED -> JOINED | | +2023-05-18T11:52:34.738484Z | got SST from cluster1-0 | +2023-05-18T11:52:34.738535Z JOINED -> SYNCED | | +2023-05-18T11:56:00.169861Z | preparing SST backup | +2023-05-18T11:56:41.702322Z | wsrep recovery | +2023-05-18T11:56:41.711514Z | JOINER -> JOINED | +2023-05-18T11:56:43.629286Z | JOINED -> SYNCED | +2023-05-18T11:59:36.017025Z cluster1-1 joined | | +2023-05-18T11:59:36.017115Z cluster1-2 joined | | +2023-05-18T11:59:36.017290Z | cluster1-2 joined | +2023-05-18T11:59:36.017369Z | cluster1-0 joined | +2023-05-18T11:59:36.018723Z | PRIMARY(n=3) | +2023-05-18T11:59:36.018796Z PRIMARY(n=3) | | +2023-05-18T11:59:37.168543Z cluster1-1 will resync cluster1-2 | | +2023-05-18T11:59:37.168800Z | local node will resync cluster1-2 | +2023-05-18T11:59:37.168859Z | SYNCED -> DONOR | +2023-05-18T11:59:37.175034Z | IST to cluster1-2(seqno:120925) | +2023-05-18T11:59:48.642038Z | SST to cluster1-2 | +2023-05-18T12:38:56.021324Z | finished sending IST to cluster1-2 | +2023-05-18T12:38:56.021429Z | DESYNCED -> JOINED | +2023-05-18T12:38:56.022044Z | JOINED -> SYNCED | +2023-05-18T12:38:56.022072Z cluster1-1 synced cluster1-2 | | +2023-05-18T13:02:37.116214Z received shutdown | | +2023-05-18T13:02:37.960715Z | received shutdown | +2023-05-18T13:02:47.118582Z | cluster1-2 joined | +2023-05-18T13:02:47.118714Z | cluster1-0 left | +2023-05-18T13:02:47.119168Z NON-PRIMARY(n=1) | | +2023-05-18T13:02:47.119351Z SYNCED -> OPEN | | +2023-05-18T13:02:47.119445Z OPEN -> CLOSED | | +2023-05-18T13:02:47.120061Z | cluster1-0 left | +2023-05-18T13:02:47.120131Z | PRIMARY(n=2) | +2023-05-18T13:02:48.963195Z | NON-PRIMARY(n=1) | +2023-05-18T13:02:48.963269Z | SYNCED -> OPEN | +2023-05-18T13:02:48.963302Z | OPEN -> CLOSED | +2023-05-18T13:13:25.811735Z starting(8.0.31) | | +2023-05-18T13:13:25.815075Z started(cluster) | | +2023-05-18T13:13:25.816336Z recovering gcache | | +2023-05-18T13:13:26.080257Z | starting(8.0.31) | +2023-05-18T13:13:26.089882Z | started(cluster) | +2023-05-18T13:13:26.091288Z | recovering gcache | +2023-05-18T13:13:27.262629Z | recovering gcache | +2023-05-18T13:13:28.077529Z | cluster1-2 joined | +2023-05-18T13:13:28.576887Z | CLOSED -> OPEN | +2023-05-18T13:13:28.577162Z | PRIMARY(n=2) | +2023-05-18T13:13:28.579204Z | OPEN -> PRIMARY | +2023-05-18T13:13:29.235576Z | will receive SST | +2023-05-18T13:13:29.236168Z | cluster1-2 will resync local node | +2023-05-18T13:13:29.236243Z | PRIMARY -> JOINER | +2023-05-18T13:13:29.243896Z | cluster1-2 failed to sync cluster1-1 | +2023-05-18T13:13:29.243939Z | will never receive SST, aborting | +2023-05-18T13:13:30.246178Z | terminated | +2023-05-18T13:13:30.246186Z | former SST cancelled | +2023-05-18T13:13:30Z | crash: got signal 11 | +2023-05-18T13:13:35.192700Z recovering gcache | | +2023-05-18T13:13:35.730381Z cluster1-2 joined | | +2023-05-18T13:13:36.229634Z CLOSED -> OPEN | | +2023-05-18T13:13:36.229890Z PRIMARY(n=2) | | +2023-05-18T13:13:36.231137Z OPEN -> PRIMARY | | +2023-05-18T13:13:36.894431Z will receive SST | | +2023-05-18T13:13:36.894926Z cluster1-2 will resync local node | | +2023-05-18T13:13:36.894974Z PRIMARY -> JOINER | | +2023-05-18T13:13:36.902951Z cluster1-2 failed to sync cluster1-0 | | +2023-05-18T13:13:36.903001Z will never receive SST, aborting | | +2023-05-18T13:13:37.905057Z terminated | | +2023-05-18T13:13:37.905067Z former SST cancelled | | +2023-05-18T13:13:37Z crash: got signal 11 | | +2023-05-18T13:14:38.342757Z | starting(8.0.31) | +2023-05-18T13:14:38.350243Z | started(cluster) | +2023-05-18T13:14:38.351647Z | recovering gcache | +2023-05-18T13:14:39.537857Z | recovering gcache | +2023-05-18T13:14:53.602717Z starting(8.0.31) | | +2023-05-18T13:14:53.605732Z started(cluster) | | +2023-05-18T13:14:53.606850Z recovering gcache | | +2023-05-18T13:15:01.357286Z recovering gcache | | +2023-05-18T13:15:19.910425Z | cluster1-2 joined | +2023-05-18T13:15:20.321473Z | CLOSED -> OPEN | +2023-05-18T13:15:20.321656Z | PRIMARY(n=2) | +2023-05-18T13:15:20.322983Z | OPEN -> PRIMARY | +2023-05-18T13:15:20.952208Z | will receive SST | +2023-05-18T13:15:20.952833Z | cluster1-2 will resync local node | +2023-05-18T13:15:20.952903Z | PRIMARY -> JOINER | +2023-05-18T13:15:20.983470Z | cluster1-2 failed to sync cluster1-1 | +2023-05-18T13:15:20.983544Z | will never receive SST, aborting | +2023-05-18T13:15:21.984924Z | terminated | +2023-05-18T13:15:21.984931Z | former SST cancelled | +2023-05-18T13:15:21Z | crash: got signal 11 | +2023-05-18T13:15:26.986441Z cluster1-2 joined | | +2023-05-18T13:15:27.250148Z CLOSED -> OPEN | | +2023-05-18T13:15:27.250392Z PRIMARY(n=2) | | +2023-05-18T13:15:27.251695Z OPEN -> PRIMARY | | +2023-05-18T13:15:27.928143Z will receive SST | | +2023-05-18T13:15:27.928839Z cluster1-2 will resync local node | | +2023-05-18T13:15:27.928922Z PRIMARY -> JOINER | | +2023-05-18T13:15:27.936607Z cluster1-2 failed to sync cluster1-0 | | +2023-05-18T13:15:27.936646Z will never receive SST, aborting | | +2023-05-18T13:15:28.938590Z terminated | | +2023-05-18T13:15:28.938596Z former SST cancelled | | +2023-05-18T13:15:28Z crash: got signal 11 | | +2023-05-18T13:16:42.748579Z starting(8.0.31) | | +2023-05-18T13:16:42.752047Z started(cluster) | | +2023-05-18T13:16:42.753354Z (repeated x1)recovering gcache | | +2023-05-18T13:16:44.038772Z | starting(8.0.31) | +2023-05-18T13:16:44.047631Z | started(cluster) | +2023-05-18T13:16:44.048841Z | (repeated x1)recovering gcache | +2023-05-18T13:16:53.022755Z recovering gcache | | +2023-05-18T13:16:54.404329Z | recovering gcache | +2023-05-18T13:17:18.945779Z cluster1-2 joined | | +2023-05-18T13:17:19.313660Z CLOSED -> OPEN | | +2023-05-18T13:17:19.313949Z PRIMARY(n=2) | | +2023-05-18T13:17:19.315225Z OPEN -> PRIMARY | | +2023-05-18T13:17:19.973975Z will receive SST | | +2023-05-18T13:17:19.974428Z cluster1-2 will resync local node | | +2023-05-18T13:17:19.974494Z PRIMARY -> JOINER | | +2023-05-18T13:17:20.792306Z receiving SST | | +2023-05-18T13:17:27.183483Z cluster1-1 joined | | +2023-05-18T13:17:27.183573Z cluster1-2 joined | | +2023-05-18T13:17:27.183629Z | cluster1-0 joined | +2023-05-18T13:17:27.183657Z | cluster1-2 joined | +2023-05-18T13:17:27.190242Z PRIMARY(n=3) | | +2023-05-18T13:17:27.682306Z | CLOSED -> OPEN | +2023-05-18T13:17:27.682491Z | PRIMARY(n=3) | +2023-05-18T13:17:27.683759Z | OPEN -> PRIMARY | +2023-05-18T13:17:28.333954Z | will receive SST | +2023-05-18T13:17:28.334941Z | cannot find donor | +2023-05-18T13:17:28.335316Z cluster1-1 cannot find donor | | +2023-05-18T13:17:29.336201Z | cannot find donor | +2023-05-18T13:17:29.336417Z cluster1-1 cannot find donor | | +2023-05-18T13:17:30.337512Z | (repeated x97)cannot find donor | +2023-05-18T13:17:30.337818Z (repeated x97)cluster1-1 cannot find donor | | +2023-05-18T13:19:08.457504Z | cannot find donor | +2023-05-18T13:19:08.457564Z cluster1-1 cannot find donor | | +2023-05-18T13:19:09.406525Z | timeout from donor in gtid/keyring stage | +2023-05-18T13:19:09.413762Z | SST error | +2023-05-18T13:19:09.415563Z | NON-PRIMARY(n=1) | +2023-05-18T13:19:09.415585Z cluster1-2 joined | | +2023-05-18T13:19:09.415687Z cluster1-1 left | | +2023-05-18T13:19:09.415724Z | PRIMARY -> OPEN | +2023-05-18T13:19:09.415816Z | OPEN -> CLOSED | +2023-05-18T13:19:09.416006Z | terminated | +2023-05-18T13:19:09.416019Z | former SST cancelled | +2023-05-18T13:19:09Z | crash: got signal 11 | +2023-05-18T13:19:09.417064Z cluster1-1 left | | +2023-05-18T13:19:09.417210Z PRIMARY(n=2) | | +2023-05-18T13:58:30.858063Z got SST from cluster1-2 | | +2023-05-18T13:59:48.769593Z | starting(8.0.31) | +2023-05-18T13:59:48.776695Z | started(cluster) | +2023-05-18T13:59:48.777920Z | (repeated x1)recovering gcache | +2023-05-18T13:59:59.654531Z | recovering gcache | +2023-05-18T14:00:40.229132Z cluster1-1 joined | | +2023-05-18T14:00:40.229229Z cluster1-2 joined | | +2023-05-18T14:00:40.229695Z | cluster1-0 joined | +2023-05-18T14:00:40.229722Z | cluster1-2 joined | +2023-05-18T14:00:40.230585Z PRIMARY(n=3) | | +2023-05-18T14:00:40.728600Z | CLOSED -> OPEN | +2023-05-18T14:00:40.728799Z | PRIMARY(n=3) | +2023-05-18T14:00:40.730137Z | OPEN -> PRIMARY | +2023-05-18T14:00:41.369410Z | will receive SST | +2023-05-18T14:00:41.369859Z cluster1-2 will resync cluster1-1 | | +2023-05-18T14:00:41.370272Z | cluster1-2 will resync local node | +2023-05-18T14:00:41.370311Z | PRIMARY -> JOINER | +2023-05-18T14:00:42.195051Z | receiving SST | +2023-05-18T14:01:56.927693Z preparing SST backup | | +2023-05-18T14:02:35.121591Z wsrep recovery | | +2023-05-18T14:02:35.186713Z JOINER -> JOINED | | +2023-05-18T14:02:37.197966Z JOINED -> SYNCED | | +2023-05-18T14:41:35.276414Z cluster1-2 synced cluster1-1 | | +2023-05-18T14:41:35.276839Z | got SST from cluster1-2 | +2023-05-18T14:45:20.989367Z | preparing SST backup | +2023-05-18T14:46:02.034943Z | wsrep recovery | +2023-05-18T14:46:02.049460Z | JOINER -> JOINED | +2023-05-18T14:46:03.716487Z | JOINED -> SYNCED | +2023-05-19T02:00:37.805239Z garb joined | | +2023-05-19T02:00:37.805339Z cluster1-1 joined | | +2023-05-19T02:00:37.805365Z cluster1-2 joined | | +2023-05-19T02:00:37.805571Z | garb joined | +2023-05-19T02:00:37.805636Z | cluster1-0 joined | +2023-05-19T02:00:37.805652Z | cluster1-2 joined | +2023-05-19T02:00:37.806917Z PRIMARY(n=4) | | +2023-05-19T02:00:37.806923Z | PRIMARY(n=4) | +2023-05-19T02:00:38.271977Z cluster1-2 will resync garb | | +2023-05-19T02:00:38.272185Z | cluster1-2 will resync garb | +2023-05-19T02:44:49.991765Z cluster1-2 synced garb | | +2023-05-19T02:44:49.991801Z | cluster1-2 synced garb | +2023-05-19T02:44:52.007874Z cluster1-1 joined | | +2023-05-19T02:44:52.007893Z cluster1-2 joined | | +2023-05-19T02:44:52.007915Z garb left | | +2023-05-19T02:44:52.009049Z | cluster1-0 joined | +2023-05-19T02:44:52.009063Z | cluster1-2 joined | +2023-05-19T02:44:52.009071Z | garb left | +2023-05-19T02:44:52.010059Z | garb left | +2023-05-19T02:44:52.010070Z garb left | | +2023-05-19T02:44:52.010131Z | PRIMARY(n=3) | +2023-05-19T02:44:52.010155Z PRIMARY(n=3) | | +2023-05-19T02:50:07.120064Z received shutdown | | +2023-05-19T02:50:17.122926Z | cluster1-2 joined | +2023-05-19T02:50:17.123053Z | cluster1-0 left | +2023-05-19T02:50:17.123274Z NON-PRIMARY(n=1) | | +2023-05-19T02:50:17.123461Z SYNCED -> OPEN | | +2023-05-19T02:50:17.123560Z OPEN -> CLOSED | | +2023-05-19T02:50:17.124167Z | cluster1-0 left | +2023-05-19T02:50:17.124228Z | PRIMARY(n=2) | +2023-05-19T03:00:44.215170Z starting(8.0.31) | | +2023-05-19T03:00:44.218802Z started(cluster) | | +2023-05-19T03:00:44.219956Z recovering gcache | | +2023-05-19T03:00:47.302675Z recovering gcache | | +2023-05-19T03:00:47.486695Z | cluster1-0 joined | +2023-05-19T03:00:47.486729Z | cluster1-2 joined | +2023-05-19T03:00:47.486876Z cluster1-1 joined | | +2023-05-19T03:00:47.486901Z cluster1-2 joined | | +2023-05-19T03:00:47.487789Z | PRIMARY(n=3) | +2023-05-19T03:00:47.887639Z CLOSED -> OPEN | | +2023-05-19T03:00:47.887884Z PRIMARY(n=3) | | +2023-05-19T03:00:47.890175Z OPEN -> PRIMARY | | +2023-05-19T03:00:48.533857Z will receive SST | | +2023-05-19T03:00:48.534320Z | local node will resync cluster1-0 | +2023-05-19T03:00:48.534355Z | SYNCED -> DONOR | +2023-05-19T03:00:48.534431Z cluster1-1 will resync local node | | +2023-05-19T03:00:48.534500Z PRIMARY -> JOINER | | +2023-05-19T03:00:48.540245Z | IST to cluster1-0(seqno:666332) | +2023-05-19T03:00:49.352397Z receiving SST | | +2023-05-19T03:00:59.993379Z | SST to cluster1-0 | +2023-05-19T03:39:14.162296Z | finished sending IST to cluster1-0 | +2023-05-19T03:39:14.162325Z | DESYNCED -> JOINED | +2023-05-19T03:39:14.162825Z | JOINED -> SYNCED | +2023-05-19T03:39:14.162862Z got SST from cluster1-1 | | +2023-05-19T03:42:34.536980Z preparing SST backup | | +2023-05-19T03:43:12.409922Z wsrep recovery | | +2023-05-19T03:43:12.417929Z JOINER -> JOINED | | +2023-05-19T03:43:13.728480Z JOINED -> SYNCED | | +2023-05-20T02:00:39.121171Z garb joined | | +2023-05-20T02:00:39.121232Z cluster1-1 joined | | +2023-05-20T02:00:39.121242Z cluster1-2 joined | | +2023-05-20T02:00:39.121260Z | garb joined | +2023-05-20T02:00:39.121310Z | cluster1-0 joined | +2023-05-20T02:00:39.121318Z | cluster1-2 joined | +2023-05-20T02:00:40.123638Z PRIMARY(n=4) | | +2023-05-20T02:00:40.123709Z | PRIMARY(n=4) | +2023-05-20T02:00:40.146578Z cluster1-2 will resync garb | | +2023-05-20T02:00:40.146782Z | cluster1-2 will resync garb | +2023-05-20T02:42:44.902691Z | cluster1-2 synced garb | +2023-05-20T02:42:44.902770Z cluster1-2 synced garb | | +2023-05-20T02:42:47.269884Z | cluster1-0 joined | +2023-05-20T02:42:47.269976Z | cluster1-2 joined | +2023-05-20T02:42:47.270003Z | garb left | +2023-05-20T02:42:47.270306Z cluster1-1 joined | | +2023-05-20T02:42:47.270430Z cluster1-2 joined | | +2023-05-20T02:42:47.270476Z garb left | | +2023-05-20T02:42:47.271539Z garb left | | +2023-05-20T02:42:47.271582Z PRIMARY(n=3) | | +2023-05-20T02:42:47.271594Z | garb left | +2023-05-20T02:42:47.271625Z | PRIMARY(n=3) | +2023-05-21T00:53:38.824876Z cluster1-1 joined | | +2023-05-21T00:53:38.824946Z | cluster1-0 joined | +2023-05-21T00:53:38.825112Z | cluster1-2 left | +2023-05-21T00:53:38.825142Z cluster1-2 left | | +2023-05-21T00:53:38.826164Z cluster1-2 left | | +2023-05-21T00:53:38.826309Z PRIMARY(n=2) | | +2023-05-21T00:53:38.826696Z | cluster1-2 left | +2023-05-21T00:53:38.826836Z | PRIMARY(n=2) | +2023-05-21T00:54:28.830096Z cluster1-2 joined | | +2023-05-21T00:54:28.830142Z | cluster1-2 joined | +2023-05-21T00:54:28.830173Z cluster1-1 joined | | +2023-05-21T00:54:28.830199Z | cluster1-0 joined | +2023-05-21T00:54:28.831682Z | PRIMARY(n=3) | +2023-05-21T00:54:28.831694Z PRIMARY(n=3) | | +2023-05-21T00:54:29.949424Z cluster1-1 will resync cluster1-2 | | +2023-05-21T00:54:29.949513Z | local node will resync cluster1-2 | +2023-05-21T00:54:29.949596Z | SYNCED -> DONOR | +2023-05-21T00:54:29.963765Z | IST to cluster1-2(seqno:2503896) | +2023-05-21T00:54:30.390475Z | IST will be used | +2023-05-21T00:54:31.413573Z cluster1-1 synced cluster1-2 | | +2023-05-21T00:54:31.413660Z | finished sending IST to cluster1-2 | +2023-05-21T00:54:31.413740Z | DESYNCED -> JOINED | +2023-05-21T00:54:31.414346Z | JOINED -> SYNCED | +2023-05-21T00:54:43.888619Z | received shutdown | +2023-05-21T00:54:53.891429Z cluster1-2 joined | | +2023-05-21T00:54:53.891553Z cluster1-1 left | | +2023-05-21T00:54:53.891588Z | NON-PRIMARY(n=1) | +2023-05-21T00:54:53.891740Z | SYNCED -> OPEN | +2023-05-21T00:54:53.891837Z | OPEN -> CLOSED | +2023-05-21T00:54:53.893441Z cluster1-1 left | | +2023-05-21T00:54:53.893497Z PRIMARY(n=2) | | +2023-05-21T00:55:15.214858Z | shutdown complete | +2023-05-21T00:55:33.793456Z | starting(8.0.31) | +2023-05-21T00:55:33.802082Z | started(cluster) | +2023-05-21T00:55:33.803486Z | recovering gcache | +2023-05-21T00:55:34.512593Z | recovering gcache | + 10.16.27.98 + (node ip) + V + 10.16.27.149 +2023-05-21T00:55:35.115170Z cluster1-2 joined | | +2023-05-21T00:55:35.115253Z cluster1-1 joined | | +2023-05-21T00:55:35.115506Z | cluster1-2 joined | +2023-05-21T00:55:35.115531Z | cluster1-0 joined | +2023-05-21T00:55:35.116524Z PRIMARY(n=3) | | +2023-05-21T00:55:35.614653Z | CLOSED -> OPEN | +2023-05-21T00:55:35.614852Z | PRIMARY(n=3) | +2023-05-21T00:55:35.615886Z | OPEN -> PRIMARY | +2023-05-21T00:55:36.251244Z | will receive IST(seqno:2504601) | +2023-05-21T00:55:36.252039Z local node will resync cluster1-1 | | +2023-05-21T00:55:36.252124Z SYNCED -> DONOR | | +2023-05-21T00:55:36.252155Z | cluster1-0 will resync local node | +2023-05-21T00:55:36.252188Z | PRIMARY -> JOINER | +2023-05-21T00:55:36.266261Z IST to cluster1-1(seqno:2504601) | | +2023-05-21T00:55:36.710320Z IST will be used | | +2023-05-21T00:55:37.727377Z finished sending IST to cluster1-1 | | +2023-05-21T00:55:37.727418Z DESYNCED -> JOINED | | +2023-05-21T00:55:37.727472Z | got IST from cluster1-0 | +2023-05-21T00:55:37.727951Z JOINED -> SYNCED | | +2023-05-21T00:55:45.002685Z | wsrep recovery | +2023-05-21T00:55:45.288182Z | IST received(seqno:2504601) | +2023-05-21T00:55:45.288875Z | JOINER -> JOINED | +2023-05-21T00:55:45.289507Z | JOINED -> SYNCED | +2023-05-21T00:55:54.030991Z received shutdown | | +2023-05-21T00:56:04.033823Z | cluster1-2 joined | +2023-05-21T00:56:04.033845Z NON-PRIMARY(n=1) | | +2023-05-21T00:56:04.033955Z | cluster1-0 left | +2023-05-21T00:56:04.034007Z SYNCED -> OPEN | | +2023-05-21T00:56:04.034129Z OPEN -> CLOSED | | +2023-05-21T00:56:04.035594Z | cluster1-0 left | +2023-05-21T00:56:04.035622Z | PRIMARY(n=2) | +2023-05-21T00:56:31.600421Z shutdown complete | | +2023-05-21T00:56:51.547391Z starting(8.0.31) | | +2023-05-21T00:56:51.551022Z started(cluster) | | +2023-05-21T00:56:51.552128Z recovering gcache | | +2023-05-21T00:56:52.187726Z recovering gcache | | + 10.16.29.188 + (node ip) + V + 10.16.29.14 +2023-05-21T00:56:52.766452Z cluster1-2 joined | | +2023-05-21T00:56:52.766462Z cluster1-1 joined | | +2023-05-21T00:56:52.766616Z | cluster1-2 joined | +2023-05-21T00:56:52.766720Z | cluster1-0 joined | +2023-05-21T00:56:52.768230Z | PRIMARY(n=3) | +2023-05-21T00:56:53.265778Z CLOSED -> OPEN | | +2023-05-21T00:56:53.265935Z PRIMARY(n=3) | | +2023-05-21T00:56:53.267175Z OPEN -> PRIMARY | | +2023-05-21T00:56:53.820470Z will receive IST(seqno:2505347) | | +2023-05-21T00:56:53.821936Z cluster1-1 will resync local node | | +2023-05-21T00:56:53.822007Z PRIMARY -> JOINER | | +2023-05-21T00:56:53.822068Z | local node will resync cluster1-0 | +2023-05-21T00:56:53.822153Z | SYNCED -> DONOR | +2023-05-21T00:56:53.840165Z | IST to cluster1-0(seqno:2505347) | +2023-05-21T00:56:54.243247Z | IST will be used | +2023-05-21T00:56:55.263953Z | finished sending IST to cluster1-0 | +2023-05-21T00:56:55.264015Z | DESYNCED -> JOINED | +2023-05-21T00:56:55.264113Z got IST from cluster1-1 | | +2023-05-21T00:56:55.264876Z | JOINED -> SYNCED | +2023-05-21T00:57:01.549618Z wsrep recovery | | +2023-05-21T00:57:01.890216Z IST received(seqno:2505347) | | +2023-05-21T00:57:01.891209Z JOINER -> JOINED | | +2023-05-21T00:57:01.891813Z JOINED -> SYNCED | | +2023-05-21T01:19:12.465768Z | cluster1-0 joined | +2023-05-21T01:19:12.465839Z cluster1-1 joined | | +2023-05-21T01:19:12.465854Z | cluster1-2 left | +2023-05-21T01:19:12.465991Z cluster1-2 left | | +2023-05-21T01:19:12.467253Z | cluster1-2 left | +2023-05-21T01:19:12.467317Z | PRIMARY(n=2) | +2023-05-21T01:19:12.467691Z cluster1-2 left | | +2023-05-21T01:19:12.467846Z PRIMARY(n=2) | | +2023-05-21T01:20:08.164581Z cluster1-1 joined | | +2023-05-21T01:20:08.164607Z | cluster1-0 joined | +2023-05-21T01:20:08.164666Z cluster1-2 joined | | +2023-05-21T01:20:08.164706Z | cluster1-2 joined | +2023-05-21T01:20:08.166058Z PRIMARY(n=3) | | +2023-05-21T01:20:08.166139Z | PRIMARY(n=3) | +2023-05-21T01:20:09.397398Z cluster1-1 will resync cluster1-2 | | +2023-05-21T01:20:09.397424Z | local node will resync cluster1-2 | +2023-05-21T01:20:09.397514Z | SYNCED -> DONOR | +2023-05-21T01:20:09.409969Z | IST to cluster1-2(seqno:2521738) | +2023-05-21T01:20:09.824497Z | IST will be used | +2023-05-21T01:20:10.843020Z cluster1-1 synced cluster1-2 | | +2023-05-21T01:20:10.843044Z | finished sending IST to cluster1-2 | +2023-05-21T01:20:10.843128Z | DESYNCED -> JOINED | +2023-05-21T01:20:10.843767Z | JOINED -> SYNCED | +2023-05-21T01:20:22.596139Z | received shutdown | +2023-05-21T01:20:32.598567Z cluster1-2 joined | | +2023-05-21T01:20:32.598720Z | NON-PRIMARY(n=1) | +2023-05-21T01:20:32.598723Z cluster1-1 left | | +2023-05-21T01:20:32.598878Z | SYNCED -> OPEN | +2023-05-21T01:20:32.598980Z | OPEN -> CLOSED | +2023-05-21T01:20:32.599998Z cluster1-1 left | | +2023-05-21T01:20:32.600066Z PRIMARY(n=2) | | +2023-05-21T01:20:52.869609Z | shutdown complete | +2023-05-21T01:21:11.409415Z | starting(8.0.31) | +2023-05-21T01:21:11.419588Z | started(cluster) | +2023-05-21T01:21:11.421017Z | recovering gcache | +2023-05-21T01:21:12.148458Z | recovering gcache | + 10.16.27.149 + (node ip) + V + 10.16.27.203 +2023-05-21T01:21:12.754120Z cluster1-2 joined | | +2023-05-21T01:21:12.754144Z | cluster1-0 joined | +2023-05-21T01:21:12.754158Z | cluster1-2 joined | +2023-05-21T01:21:12.754192Z cluster1-1 joined | | +2023-05-21T01:21:12.755542Z PRIMARY(n=3) | | +2023-05-21T01:21:13.253595Z | CLOSED -> OPEN | +2023-05-21T01:21:13.253786Z | PRIMARY(n=3) | +2023-05-21T01:21:13.254822Z | OPEN -> PRIMARY | +2023-05-21T01:21:13.830568Z | will receive IST(seqno:2522685) | +2023-05-21T01:21:13.831224Z cluster1-2 will resync cluster1-1 | | +2023-05-21T01:21:13.831364Z | cluster1-2 will resync local node | +2023-05-21T01:21:13.831439Z | PRIMARY -> JOINER | +2023-05-21T01:21:15.333851Z cluster1-2 synced cluster1-1 | | +2023-05-21T01:21:15.333858Z | got IST from cluster1-2 | +2023-05-21T01:21:22.288131Z | wsrep recovery | +2023-05-21T01:21:22.898839Z | IST received(seqno:2522685) | +2023-05-21T01:21:22.899619Z | JOINER -> JOINED | +2023-05-21T01:21:23.029024Z | JOINED -> SYNCED | +2023-05-21T01:21:33.487761Z received shutdown | | +2023-05-21T01:21:43.490218Z | cluster1-2 joined | +2023-05-21T01:21:43.490347Z NON-PRIMARY(n=1) | | +2023-05-21T01:21:43.490351Z | cluster1-0 left | +2023-05-21T01:21:43.490461Z SYNCED -> OPEN | | +2023-05-21T01:21:43.490528Z OPEN -> CLOSED | | +2023-05-21T01:21:43.492282Z | cluster1-0 left | +2023-05-21T01:21:43.492343Z | PRIMARY(n=2) | +2023-05-21T01:22:09.957644Z shutdown complete | | +2023-05-21T01:22:28.363578Z starting(8.0.31) | | +2023-05-21T01:22:28.367473Z started(cluster) | | +2023-05-21T01:22:28.368588Z recovering gcache | | +2023-05-21T01:22:29.021216Z recovering gcache | | + 10.16.29.14 + (node ip) + V + 10.16.29.241 +2023-05-21T01:22:29.152458Z | cluster1-0 joined | +2023-05-21T01:22:29.152542Z | cluster1-2 joined | +2023-05-21T01:22:29.152837Z cluster1-2 joined | | +2023-05-21T01:22:29.152861Z cluster1-1 joined | | +2023-05-21T01:22:29.153988Z | PRIMARY(n=3) | +2023-05-21T01:22:29.618317Z CLOSED -> OPEN | | +2023-05-21T01:22:29.618481Z PRIMARY(n=3) | | +2023-05-21T01:22:29.620709Z OPEN -> PRIMARY | | +2023-05-21T01:22:30.266257Z will receive IST(seqno:2523752) | | +2023-05-21T01:22:30.267198Z cluster1-1 will resync local node | | +2023-05-21T01:22:30.267264Z | local node will resync cluster1-0 | +2023-05-21T01:22:30.267267Z PRIMARY -> JOINER | | +2023-05-21T01:22:30.267342Z | SYNCED -> DONOR | +2023-05-21T01:22:30.284151Z | IST to cluster1-0(seqno:2523752) | +2023-05-21T01:22:30.690731Z | IST will be used | +2023-05-21T01:22:31.707614Z got IST from cluster1-1 | | +2023-05-21T01:22:31.707646Z | finished sending IST to cluster1-0 | +2023-05-21T01:22:31.707711Z | DESYNCED -> JOINED | +2023-05-21T01:22:31.708193Z | JOINED -> SYNCED | +2023-05-21T01:22:38.451890Z wsrep recovery | | +2023-05-21T01:22:38.880335Z IST received(seqno:2523752) | | +2023-05-21T01:22:38.882383Z JOINER -> JOINED | | +2023-05-21T01:22:38.883539Z JOINED -> SYNCED | | +2023-05-21T03:00:20.427423Z | cluster1-0 joined | +2023-05-21T03:00:20.427494Z | garb joined | +2023-05-21T03:00:20.427533Z | cluster1-2 joined | +2023-05-21T03:00:20.427567Z garb joined | | +2023-05-21T03:00:20.427614Z cluster1-2 joined | | +2023-05-21T03:00:20.427622Z cluster1-1 joined | | +2023-05-21T03:00:20.429353Z | PRIMARY(n=4) | +2023-05-21T03:00:20.429883Z PRIMARY(n=4) | | +2023-05-21T03:00:20.928257Z | cluster1-2 will resync garb | +2023-05-21T03:00:20.928631Z cluster1-2 will resync garb | | +2023-05-21T03:42:29.775490Z | cluster1-2 synced garb | +2023-05-21T03:42:29.776133Z cluster1-2 synced garb | | +2023-05-21T03:42:31.791064Z | cluster1-0 joined | +2023-05-21T03:42:31.791148Z | cluster1-2 joined | +2023-05-21T03:42:31.791208Z | garb left | +2023-05-21T03:42:31.791969Z cluster1-2 joined | | +2023-05-21T03:42:31.792056Z cluster1-1 joined | | +2023-05-21T03:42:31.792084Z garb left | | +2023-05-21T03:42:31.792896Z | garb left | +2023-05-21T03:42:31.792966Z | PRIMARY(n=3) | +2023-05-21T03:42:31.793522Z garb left | | +2023-05-21T03:42:31.793616Z PRIMARY(n=3) | | +2023-05-22T03:00:18.103104Z cluster1-2 joined | | +2023-05-22T03:00:18.103142Z cluster1-1 joined | | +2023-05-22T03:00:18.103149Z garb joined | | +2023-05-22T03:00:18.103256Z | cluster1-0 joined | +2023-05-22T03:00:18.103420Z | cluster1-2 joined | +2023-05-22T03:00:18.103448Z | garb joined | +2023-05-22T03:00:18.104955Z PRIMARY(n=4) | | +2023-05-22T03:00:18.105186Z | PRIMARY(n=4) | +2023-05-22T03:00:18.604335Z cluster1-2 will resync garb | | +2023-05-22T03:00:18.604404Z | cluster1-2 will resync garb | +2023-05-22T03:42:17.472102Z | cluster1-2 synced garb | +2023-05-22T03:42:17.472302Z cluster1-2 synced garb | | +2023-05-22T03:42:19.981973Z cluster1-2 joined | | +2023-05-22T03:42:19.982069Z cluster1-1 joined | | +2023-05-22T03:42:19.982070Z | cluster1-0 joined | +2023-05-22T03:42:19.982096Z garb left | | +2023-05-22T03:42:19.982108Z | cluster1-2 joined | +2023-05-22T03:42:19.982132Z | garb left | +2023-05-22T03:42:19.986126Z | garb left | +2023-05-22T03:42:19.986197Z | PRIMARY(n=3) | +2023-05-22T03:42:19.986489Z garb left | | +2023-05-22T03:42:19.986558Z PRIMARY(n=3) | | +2023-05-22T05:00:38.651931Z garb joined | | +2023-05-22T05:00:38.651990Z | cluster1-0 joined | +2023-05-22T05:00:38.652028Z cluster1-2 joined | | +2023-05-22T05:00:38.652053Z cluster1-1 joined | | +2023-05-22T05:00:38.652099Z | garb joined | +2023-05-22T05:00:38.652124Z | cluster1-2 joined | +2023-05-22T05:00:38.654074Z | PRIMARY(n=4) | +2023-05-22T05:00:38.654180Z PRIMARY(n=4) | | +2023-05-22T05:00:39.152013Z cluster1-2 will resync garb | | +2023-05-22T05:00:39.152105Z | cluster1-2 will resync garb | +2023-05-22T05:42:27.093203Z cluster1-2 synced garb | | +2023-05-22T05:42:27.093298Z | cluster1-2 synced garb | +2023-05-22T05:42:29.265224Z cluster1-2 joined | | +2023-05-22T05:42:29.265242Z cluster1-1 joined | | +2023-05-22T05:42:29.265253Z garb left | | +2023-05-22T05:42:29.265680Z | cluster1-0 joined | +2023-05-22T05:42:29.265724Z | cluster1-2 joined | +2023-05-22T05:42:29.265751Z | garb left | +2023-05-22T05:42:29.266637Z garb left | | +2023-05-22T05:42:29.266704Z | garb left | +2023-05-22T05:42:29.266717Z PRIMARY(n=3) | | +2023-05-22T05:42:29.266780Z | PRIMARY(n=3) | +2023-05-23T03:00:20.329378Z | cluster1-0 joined | +2023-05-23T03:00:20.329435Z garb joined | | +2023-05-23T03:00:20.329467Z | garb joined | +2023-05-23T03:00:20.329491Z | cluster1-2 joined | +2023-05-23T03:00:20.329519Z cluster1-2 joined | | +2023-05-23T03:00:20.329543Z cluster1-1 joined | | +2023-05-23T03:00:20.330917Z PRIMARY(n=4) | | +2023-05-23T03:00:20.330937Z | PRIMARY(n=4) | +2023-05-23T03:00:20.830202Z | cluster1-2 will resync garb | +2023-05-23T03:00:20.830302Z cluster1-2 will resync garb | | +2023-05-23T03:42:33.009984Z cluster1-2 synced garb | | +2023-05-23T03:42:33.010045Z | cluster1-2 synced garb | +2023-05-23T03:42:35.008485Z cluster1-2 joined | | +2023-05-23T03:42:35.008497Z cluster1-1 joined | | +2023-05-23T03:42:35.008504Z garb left | | +2023-05-23T03:42:35.009285Z | cluster1-0 joined | +2023-05-23T03:42:35.009336Z | cluster1-2 joined | +2023-05-23T03:42:35.009360Z | garb left | +2023-05-23T03:42:35.010350Z garb left | | +2023-05-23T03:42:35.010383Z PRIMARY(n=3) | | +2023-05-23T03:42:35.024977Z | garb left | +2023-05-23T03:42:35.025045Z | PRIMARY(n=3) | +2023-05-24T03:00:21.217120Z | cluster1-0 joined | +2023-05-24T03:00:21.217214Z | garb joined | +2023-05-24T03:00:21.217238Z | cluster1-2 joined | +2023-05-24T03:00:21.217414Z garb joined | | +2023-05-24T03:00:21.217519Z cluster1-2 joined | | +2023-05-24T03:00:21.217546Z cluster1-1 joined | | +2023-05-24T03:00:21.218871Z | PRIMARY(n=4) | +2023-05-24T03:00:21.218947Z PRIMARY(n=4) | | +2023-05-24T03:00:21.717835Z | cluster1-2 will resync garb | +2023-05-24T03:00:21.718081Z cluster1-2 will resync garb | | +2023-05-24T03:45:11.980179Z cluster1-2 synced garb | | +2023-05-24T03:45:11.980189Z | cluster1-2 synced garb | +2023-05-24T03:45:14.589728Z cluster1-2 joined | | +2023-05-24T03:45:14.589729Z | cluster1-0 joined | +2023-05-24T03:45:14.589749Z cluster1-1 joined | | +2023-05-24T03:45:14.589763Z garb left | | +2023-05-24T03:45:14.589767Z | cluster1-2 joined | +2023-05-24T03:45:14.589789Z | garb left | +2023-05-24T03:45:14.591467Z | garb left | +2023-05-24T03:45:14.591482Z garb left | | +2023-05-24T03:45:14.591503Z | PRIMARY(n=3) | +2023-05-24T03:45:14.591557Z PRIMARY(n=3) | | +2023-05-24T08:21:16.472713Z | cluster1-0 joined | +2023-05-24T08:21:16.472872Z cluster1-1 joined | | +2023-05-24T08:21:16.472985Z cluster1-2 left | | +2023-05-24T08:21:16.474136Z | cluster1-2 left | +2023-05-24T08:21:16.475291Z cluster1-2 left | | +2023-05-24T08:21:16.475463Z | cluster1-2 left | +2023-05-24T08:21:16.475483Z PRIMARY(n=2) | | +2023-05-24T08:21:16.475528Z | PRIMARY(n=2) | +2023-05-24T08:22:01.125588Z | cluster1-0 joined | +2023-05-24T08:22:01.125674Z | cluster1-2 joined | +2023-05-24T08:22:01.125862Z cluster1-2 joined | | +2023-05-24T08:22:01.125950Z cluster1-1 joined | | +2023-05-24T08:22:01.127244Z | PRIMARY(n=3) | +2023-05-24T08:22:01.127387Z PRIMARY(n=3) | | +2023-05-24T08:22:02.311565Z | local node will resync cluster1-2 | +2023-05-24T08:22:02.311647Z | SYNCED -> DONOR | +2023-05-24T08:22:02.311766Z cluster1-1 will resync cluster1-2 | | +2023-05-24T08:22:02.325257Z | IST to cluster1-2(seqno:5144165) | +2023-05-24T08:22:02.748089Z | IST will be used | +2023-05-24T08:22:03.764239Z | finished sending IST to cluster1-2 | +2023-05-24T08:22:03.764294Z | DESYNCED -> JOINED | +2023-05-24T08:22:03.764584Z cluster1-1 synced cluster1-2 | | +2023-05-24T08:22:03.764839Z | JOINED -> SYNCED | +2023-05-24T08:32:37.472913Z received shutdown | | +2023-05-24T08:42:31.487071Z | cluster1-2 joined | +2023-05-24T08:42:31.487146Z | cluster1-0 left | +2023-05-24T08:42:31.487732Z NON-PRIMARY(n=1) | | +2023-05-24T08:42:31.487796Z SYNCED -> OPEN | | +2023-05-24T08:42:31.487828Z OPEN -> CLOSED | | +2023-05-24T08:42:31.488231Z | cluster1-0 left | +2023-05-24T08:42:31.488252Z | PRIMARY(n=2) | +2023-05-24T08:43:00.498377Z starting(8.0.31) | | +2023-05-24T08:43:00.504399Z started(cluster) | | +2023-05-24T08:43:00.505651Z recovering gcache | | +2023-05-24T08:43:01.141047Z recovering gcache | | +2023-05-24T08:43:01.386744Z | cluster1-0 joined | +2023-05-24T08:43:01.386792Z | cluster1-2 joined | +2023-05-24T08:43:01.387024Z cluster1-2 joined | | +2023-05-24T08:43:01.387034Z cluster1-1 joined | | +2023-05-24T08:43:01.387588Z | PRIMARY(n=3) | +2023-05-24T08:43:01.712895Z CLOSED -> OPEN | | +2023-05-24T08:43:01.713165Z PRIMARY(n=3) | | +2023-05-24T08:43:01.714718Z OPEN -> PRIMARY | | +2023-05-24T08:43:02.417808Z will receive IST(seqno:5153041) | | +2023-05-24T08:43:02.418439Z | local node will resync cluster1-0 | +2023-05-24T08:43:02.418491Z | SYNCED -> DONOR | +2023-05-24T08:43:02.418682Z cluster1-1 will resync local node | | +2023-05-24T08:43:02.418709Z PRIMARY -> JOINER | | +2023-05-24T08:43:02.432861Z | IST to cluster1-0(seqno:5153041) | +2023-05-24T08:43:02.859610Z | IST will be used | +2023-05-24T08:43:03.879518Z | finished sending IST to cluster1-0 | +2023-05-24T08:43:03.879590Z | DESYNCED -> JOINED | +2023-05-24T08:43:03.879884Z got IST from cluster1-1 | | +2023-05-24T08:43:03.880269Z | JOINED -> SYNCED | +2023-05-24T08:43:11.252798Z wsrep recovery | | +2023-05-24T08:43:11.779173Z IST received(seqno:5153041) | | +2023-05-24T08:43:11.779858Z JOINER -> JOINED | | +2023-05-24T08:43:11.780506Z JOINED -> SYNCED | | +2023-05-24T08:51:31.191288Z (repeated x14127)too many connections | | +2023-05-24T08:54:25.857147Z | (repeated x1236)too many connections | +2023-05-24T08:55:40.532333Z | too many connections | +2023-05-24T08:55:40.543569Z too many connections | | +2023-05-24T08:55:40.583937Z | (repeated x54)too many connections | +2023-05-24T08:55:40.589334Z (repeated x139)too many connections | | +2023-05-24T08:55:42.449249Z too many connections | | +2023-05-24T08:55:42.462964Z cluster1-2 suspected to be down | | +2023-05-24T08:55:42.473404Z (repeated x87)too many connections | | +2023-05-24T08:55:42.934639Z | too many connections | +2023-05-24T08:55:42.963220Z | cluster1-2 suspected to be down | +2023-05-24T08:55:42.982146Z | (repeated x5)too many connections | +2023-05-24T08:55:43.411608Z | too many connections | +2023-05-24T08:55:43.460587Z too many connections | | +2023-05-24T08:55:43.463902Z | cluster1-0 joined | +2023-05-24T08:55:43.464038Z cluster1-1 joined | | +2023-05-24T08:55:43.464983Z cluster1-2 left | | +2023-05-24T08:55:43.465054Z PRIMARY(n=2) | | +2023-05-24T08:55:43.466667Z | cluster1-2 left | +2023-05-24T08:55:43.466742Z | PRIMARY(n=2) | +2023-05-24T08:55:43.478867Z (repeated x2618)too many connections | | +2023-05-24T08:55:43.549775Z | (repeated x914)too many connections | +2023-05-24T08:56:19.200275Z | too many connections | +2023-05-24T08:56:25.582987Z too many connections | | +2023-05-24T08:56:25.600196Z (repeated x26)too many connections | | +2023-05-24T08:56:26.060649Z too many connections | | +2023-05-24T08:56:26.079279Z | cluster1-0 joined | +2023-05-24T08:56:26.079318Z | cluster1-2 joined | +2023-05-24T08:56:26.079533Z cluster1-2 joined | | +2023-05-24T08:56:26.079603Z cluster1-1 joined | | +2023-05-24T08:56:26.080575Z | PRIMARY(n=3) | +2023-05-24T08:56:26.081031Z PRIMARY(n=3) | | +2023-05-24T08:56:26.083864Z too many connections | | +2023-05-24T08:56:26.089732Z (repeated x42)too many connections | | +2023-05-24T08:56:26.770395Z too many connections | | +2023-05-24T08:56:26.780586Z | local node will resync cluster1-2 | +2023-05-24T08:56:26.780678Z | SYNCED -> DONOR | +2023-05-24T08:56:26.780686Z cluster1-1 will resync cluster1-2 | | +2023-05-24T08:56:26.794328Z | IST to cluster1-2(seqno:5154413) | +2023-05-24T08:56:26.799475Z (repeated x94)too many connections | | +2023-05-24T08:56:27.242147Z | IST will be used | +2023-05-24T08:56:28.245207Z too many connections | | +2023-05-24T08:56:28.259779Z | finished sending IST to cluster1-2 | +2023-05-24T08:56:28.259847Z | DESYNCED -> JOINED | +2023-05-24T08:56:28.259965Z cluster1-1 synced cluster1-2 | | +2023-05-24T08:56:28.260407Z | JOINED -> SYNCED | +2023-05-24T08:56:28.284935Z (repeated x10)too many connections | | +2023-05-24T08:56:28.704294Z too many connections | | +2023-05-24T08:56:28.808334Z (repeated x3590)too many connections | | +2023-05-24T08:57:03.500612Z | (repeated x168)too many connections | +2023-05-24T08:57:16.956296Z | too many connections | +2023-05-24T08:57:20.491581Z | received shutdown | +2023-05-24T08:57:23.761287Z too many connections | | +2023-05-19T03:42:52.432749Z starting(8.0.31)(post.processing.log) | | +2023-05-19T03:42:58.941611Z started(standalone)(post.processing.log) | | +2023-05-24T08:57:23.767800Z (repeated x7198)too many connections | | +2023-05-24T08:57:27.361076Z | too many connections | +2023-05-24T08:59:34.202664Z too many connections | | +2023-05-24T09:07:25.631121Z cluster1-1 suspected to be down | | +2023-05-24T09:07:26.131695Z cluster1-2 joined | | +2023-05-24T09:07:26.132852Z cluster1-1 left | | +2023-05-24T09:07:26.132933Z PRIMARY(n=2) | | +2023-05-24T09:08:00.080637Z | starting(8.0.31) | +2023-05-24T09:08:00.090151Z | started(cluster) | +2023-05-24T09:08:00.091405Z | recovering gcache | +2023-05-24T09:08:00.706292Z | recovering gcache | +2023-05-24T09:08:01.278958Z | CLOSED -> OPEN | +2023-05-24T09:08:01.280521Z cluster1-2 joined | | +2023-05-24T09:08:01.280524Z | cluster1-0 joined | +2023-05-24T09:08:01.280556Z | cluster1-2 joined | +2023-05-24T09:08:01.280599Z cluster1-1 joined | | +2023-05-24T09:08:01.282166Z | PRIMARY(n=3) | +2023-05-24T09:08:01.282175Z PRIMARY(n=3) | | +2023-05-24T09:08:01.283175Z | OPEN -> PRIMARY | +2023-05-24T09:08:01.941376Z | will receive IST(seqno:5155889) | +2023-05-24T09:08:01.941969Z | cluster1-2 will resync local node | +2023-05-24T09:08:01.941997Z | PRIMARY -> JOINER | +2023-05-24T09:08:01.942020Z cluster1-2 will resync cluster1-1 | | +2023-05-24T09:08:03.417354Z | got IST from cluster1-2 | +2023-05-24T09:08:03.417514Z cluster1-2 synced cluster1-1 | | +2023-05-24T09:08:09.881553Z | wsrep recovery | +2023-05-24T09:08:10.997657Z | IST received(seqno:5155889) | +2023-05-24T09:08:10.998552Z | JOINER -> JOINED | +2023-05-24T09:08:10.999115Z | JOINED -> SYNCED | +2023-05-24T09:08:57.462448Z received shutdown | | +2023-05-24T09:19:02.336997Z | cluster1-0 suspected to be down | +2023-05-24T09:19:03.338014Z | cluster1-2 joined | +2023-05-24T09:19:03.340267Z | cluster1-0 left | +2023-05-24T09:19:03.340337Z | PRIMARY(n=2) | +2023-05-24T09:19:37.055707Z starting(8.0.31) | | +2023-05-24T09:19:37.071531Z started(cluster) | | +2023-05-24T09:19:37.072937Z recovering gcache | | +2023-05-24T09:19:37.704088Z recovering gcache | | +2023-05-24T09:19:38.280834Z | cluster1-0 joined | +2023-05-24T09:19:38.280937Z | cluster1-2 joined | +2023-05-24T09:19:38.280964Z cluster1-2 joined | | +2023-05-24T09:19:38.280992Z cluster1-1 joined | | +2023-05-24T09:19:38.282667Z | PRIMARY(n=3) | +2023-05-24T09:19:38.779653Z CLOSED -> OPEN | | +2023-05-24T09:19:38.779857Z PRIMARY(n=3) | | +2023-05-24T09:19:38.781851Z OPEN -> PRIMARY | | +2023-05-24T09:19:39.338473Z will receive IST(seqno:5156720) | | +2023-05-24T09:19:39.339687Z | local node will resync cluster1-0 | +2023-05-24T09:19:39.339783Z cluster1-1 will resync local node | | +2023-05-24T09:19:39.339784Z | SYNCED -> DONOR | +2023-05-24T09:19:39.339860Z PRIMARY -> JOINER | | +2023-05-24T09:19:39.359574Z | IST to cluster1-0(seqno:5156720) | +2023-05-24T09:19:39.812801Z | IST will be used | +2023-05-24T09:19:40.833367Z | finished sending IST to cluster1-0 | +2023-05-24T09:19:40.833393Z got IST from cluster1-1 | | +2023-05-24T09:19:40.833451Z | DESYNCED -> JOINED | +2023-05-24T09:19:40.834089Z | JOINED -> SYNCED | +2023-05-24T09:19:47.390495Z wsrep recovery | | +2023-05-24T09:19:47.742772Z IST received(seqno:5156720) | | +2023-05-24T09:19:47.744825Z JOINER -> JOINED | | +2023-05-24T09:19:47.745816Z JOINED -> SYNCED | | +2023-05-24T09:19:58.001359Z | (repeated x670)too many connections | +2023-05-24T09:23:58.339523Z | too many connections | +2023-05-25T03:00:18.871163Z cluster1-2 joined | | +2023-05-25T03:00:18.871225Z garb joined | | +2023-05-25T03:00:18.871235Z cluster1-1 joined | | +2023-05-25T03:00:18.871388Z | cluster1-0 joined | +2023-05-25T03:00:18.871468Z | cluster1-2 joined | +2023-05-25T03:00:18.871485Z | garb joined | +2023-05-25T03:00:19.873796Z | PRIMARY(n=4) | +2023-05-25T03:00:19.874026Z PRIMARY(n=4) | | +2023-05-25T03:00:20.363932Z cluster1-2 will resync garb | | +2023-05-25T03:00:20.363986Z | cluster1-2 will resync garb | +2023-05-25T03:44:11.626005Z | cluster1-2 synced garb | +2023-05-25T03:44:11.626112Z cluster1-2 synced garb | | +2023-05-25T03:44:13.987810Z | cluster1-0 joined | +2023-05-25T03:44:13.987866Z | cluster1-2 joined | +2023-05-25T03:44:13.987902Z | garb left | +2023-05-25T03:44:13.987908Z cluster1-2 joined | | +2023-05-25T03:44:13.987968Z cluster1-1 joined | | +2023-05-25T03:44:13.988012Z garb left | | +2023-05-25T03:44:13.988890Z | garb left | +2023-05-25T03:44:13.988956Z | PRIMARY(n=3) | +2023-05-25T03:44:13.989008Z garb left | | +2023-05-25T03:44:13.989030Z PRIMARY(n=3) | | +2023-05-25T03:48:13.523650Z cluster1-1 joined | | +2023-05-25T03:48:13.523696Z | cluster1-0 joined | +2023-05-25T03:48:13.523738Z cluster1-2 left | | +2023-05-25T03:48:13.523884Z | cluster1-2 left | +2023-05-25T03:48:13.525186Z cluster1-2 left | | +2023-05-25T03:48:13.525258Z PRIMARY(n=2) | | +2023-05-25T03:48:13.525483Z | cluster1-2 left | +2023-05-25T03:48:13.525517Z | PRIMARY(n=2) | +2023-05-25T03:49:24.329464Z | | starting(8.0.31) +2023-05-25T03:49:24.332449Z | | started(cluster) +2023-05-25T03:49:24.333454Z | | no grastate.dat file +2023-05-25T03:49:24.333464Z | | bootstrapping(empty grastate) +2023-05-25T03:49:24.333499Z | | safe_to_bootstrap: 1 +2023-05-25T03:49:24.849844Z | cluster1-0 joined | +2023-05-25T03:49:24.849938Z | cluster1-2 joined | +2023-05-25T03:49:24.850000Z cluster1-2 joined | | +2023-05-25T03:49:24.850187Z cluster1-1 joined | | +2023-05-25T03:49:24.850405Z | | cluster1-0 joined +2023-05-25T03:49:24.850417Z | | cluster1-1 joined +2023-05-25T03:49:25.852094Z | PRIMARY(n=3) | +2023-05-25T03:49:25.852441Z PRIMARY(n=3) | | +2023-05-25T03:49:26.348763Z | | CLOSED -> OPEN +2023-05-25T03:49:26.349013Z | | PRIMARY(n=3) +2023-05-25T03:49:26.350500Z | | OPEN -> PRIMARY +2023-05-25T03:49:27.004187Z | | will receive SST +2023-05-25T03:49:27.005073Z | local node will resync cluster1-2 | +2023-05-25T03:49:27.005141Z cluster1-1 will resync cluster1-2 | | +2023-05-25T03:49:27.005200Z | SYNCED -> DONOR | +2023-05-25T03:49:27.005213Z | | cluster1-1 will resync local node +2023-05-25T03:49:27.005286Z | | PRIMARY -> JOINER +2023-05-25T03:49:27.012633Z | IST to cluster1-2(seqno:5835877) | +2023-05-25T03:49:27.823189Z | | receiving SST +2023-05-25T03:49:38.513896Z | SST to cluster1-2 | +2023-05-25T04:31:22.635591Z | finished sending IST to cluster1-2 | +2023-05-25T04:31:22.635628Z | DESYNCED -> JOINED | +2023-05-25T04:31:22.635717Z | | got SST from cluster1-1 +2023-05-25T04:31:22.635767Z cluster1-1 synced cluster1-2 | | +2023-05-25T04:31:22.636094Z | JOINED -> SYNCED | +2023-05-25T04:31:22.667270Z | | preparing SST backup +2023-05-25T04:31:58.316262Z | | wsrep recovery +2023-05-25T04:31:58.337408Z | | JOINER -> JOINED +2023-05-25T04:31:59.767489Z | | JOINED -> SYNCED +2023-05-25T04:34:20.398345Z | | received shutdown +2023-05-25T04:34:30.401364Z | cluster1-0 joined | +2023-05-25T04:34:30.401445Z cluster1-1 joined | | +2023-05-25T04:34:30.401532Z | | NON-PRIMARY(n=1) +2023-05-25T04:34:30.401543Z | cluster1-2 left | +2023-05-25T04:34:30.401594Z cluster1-2 left | | +2023-05-25T04:34:30.401727Z | | SYNCED -> OPEN +2023-05-25T04:34:30.401855Z | | OPEN -> CLOSED +2023-05-25T04:34:30.403167Z cluster1-2 left | | +2023-05-25T04:34:30.403229Z PRIMARY(n=2) | | +2023-05-25T04:34:30.403506Z | cluster1-2 left | +2023-05-25T04:34:30.403555Z | PRIMARY(n=2) | +2023-05-25T04:31:38.830198Z | | starting(8.0.31)(post.processing.log) +2023-05-25T04:31:45.148850Z | | started(standalone)(post.processing.log) +2023-05-25T04:34:43.205745Z | | shutdown complete +2023-05-25T04:31:51.207735Z | | shutdown complete(post.processing.log) +2023-05-25T04:35:01.293586Z | | starting(8.0.31) +2023-05-25T04:35:01.296248Z | | started(cluster) +2023-05-25T04:35:01.297362Z | | recovering gcache +2023-05-25T04:35:01.331535Z | | recovering gcache + 10.16.28.207 + (node ip) + V + 10.16.28.213 +2023-05-25T04:35:01.848758Z | cluster1-0 joined | +2023-05-25T04:35:01.848759Z | | cluster1-0 joined +2023-05-25T04:35:01.848785Z | | cluster1-1 joined +2023-05-25T04:35:01.848845Z | cluster1-2 joined | +2023-05-25T04:35:01.848877Z cluster1-2 joined | | +2023-05-25T04:35:01.848959Z cluster1-1 joined | | +2023-05-25T04:35:01.850481Z | PRIMARY(n=3) | +2023-05-25T04:35:01.850512Z PRIMARY(n=3) | | +2023-05-25T04:35:02.347755Z | | CLOSED -> OPEN +2023-05-25T04:35:02.347927Z | | PRIMARY(n=3) +2023-05-25T04:35:02.349077Z | | OPEN -> PRIMARY +2023-05-25T04:35:02.908838Z | | will receive IST(seqno:5858164) +2023-05-25T04:35:02.909866Z | | cluster1-1 will resync local node +2023-05-25T04:35:02.909899Z | | PRIMARY -> JOINER +2023-05-25T04:35:02.909925Z cluster1-1 will resync cluster1-2 | | +2023-05-25T04:35:02.909983Z | local node will resync cluster1-2 | +2023-05-25T04:35:02.910059Z | SYNCED -> DONOR | +2023-05-25T04:35:02.921394Z | IST to cluster1-2(seqno:5858164) | +2023-05-25T04:35:03.336995Z | IST will be used | +2023-05-25T04:35:04.355833Z | finished sending IST to cluster1-2 | +2023-05-25T04:35:04.355886Z | | got IST from cluster1-1 +2023-05-25T04:35:04.355910Z | DESYNCED -> JOINED | +2023-05-25T04:35:04.355933Z cluster1-1 synced cluster1-2 | | +2023-05-25T04:35:04.356540Z | JOINED -> SYNCED | +2023-05-25T04:35:14.895241Z | | wsrep recovery +2023-05-25T04:35:15.076324Z | | IST received(seqno:5858164) +2023-05-25T04:35:15.077560Z | | JOINER -> JOINED +2023-05-25T04:35:15.078178Z | | JOINED -> SYNCED +2023-05-25T04:35:20.556329Z | received shutdown | +2023-05-25T04:35:30.559038Z | | cluster1-0 joined +2023-05-25T04:35:30.559192Z cluster1-2 joined | | +2023-05-25T04:35:30.559235Z | NON-PRIMARY(n=1) | +2023-05-25T04:35:30.559318Z cluster1-1 left | | +2023-05-25T04:35:30.559398Z | SYNCED -> OPEN | +2023-05-25T04:35:30.559515Z | OPEN -> CLOSED | +2023-05-25T04:35:30.559684Z | | cluster1-1 left +2023-05-25T04:35:30.560761Z cluster1-1 left | | +2023-05-25T04:35:30.560850Z PRIMARY(n=2) | | +2023-05-25T04:35:30.561080Z | | cluster1-1 left +2023-05-25T04:35:30.561149Z | | PRIMARY(n=2) +2023-05-25T04:35:53.568992Z | shutdown complete | +2023-05-18T14:45:54.212160Z | shutdown complete(post.processing.log) | +2023-05-25T04:36:12.725113Z | starting(8.0.31) | +2023-05-25T04:36:12.735830Z | started(cluster) | +2023-05-25T04:36:12.737161Z | recovering gcache | +2023-05-25T04:36:13.381040Z | recovering gcache | + 10.16.27.203 + (node ip) + V + 10.16.27.98 +2023-05-25T04:36:13.977726Z | | cluster1-0 joined +2023-05-25T04:36:13.977805Z | | cluster1-1 joined +2023-05-25T04:36:13.977867Z | cluster1-0 joined | +2023-05-25T04:36:13.977899Z | cluster1-2 joined | +2023-05-25T04:36:13.977932Z cluster1-2 joined | | +2023-05-25T04:36:13.978032Z cluster1-1 joined | | +2023-05-25T04:36:13.979426Z | | PRIMARY(n=3) +2023-05-25T04:36:13.979510Z PRIMARY(n=3) | | +2023-05-25T04:36:14.476933Z | CLOSED -> OPEN | +2023-05-25T04:36:14.477197Z | PRIMARY(n=3) | +2023-05-25T04:36:14.478368Z | OPEN -> PRIMARY | +2023-05-25T04:36:15.096770Z | will receive IST(seqno:5859104) | +2023-05-25T04:36:15.097710Z | cluster1-2 will resync local node | +2023-05-25T04:36:15.097740Z | PRIMARY -> JOINER | +2023-05-25T04:36:15.097743Z | | local node will resync cluster1-1 +2023-05-25T04:36:15.097810Z cluster1-2 will resync cluster1-1 | | +2023-05-25T04:36:15.097829Z | | SYNCED -> DONOR +2023-05-25T04:36:15.113859Z | | IST to cluster1-1(seqno:5859104) +2023-05-25T04:36:15.523060Z | | IST will be used +2023-05-25T04:36:16.543304Z | | finished sending IST to cluster1-1 +2023-05-25T04:36:16.543332Z | | DESYNCED -> JOINED +2023-05-25T04:36:16.543351Z | got IST from cluster1-2 | +2023-05-25T04:36:16.543389Z cluster1-2 synced cluster1-1 | | +2023-05-25T04:36:16.543843Z | | JOINED -> SYNCED +2023-05-25T04:36:29.019181Z | wsrep recovery | +2023-05-25T04:36:29.641103Z | IST received(seqno:5859104) | +2023-05-25T04:36:29.642082Z | JOINER -> JOINED | +2023-05-25T04:36:29.847276Z | JOINED -> SYNCED | +2023-05-25T04:37:01.595911Z received shutdown | | +2023-05-25T04:37:12.599050Z NON-PRIMARY(n=1) | | +2023-05-25T04:37:12.599120Z | cluster1-2 joined | +2023-05-25T04:37:12.599211Z SYNCED -> OPEN | | +2023-05-25T04:37:12.599244Z | cluster1-0 left | +2023-05-25T04:37:12.599252Z | | cluster1-1 joined +2023-05-25T04:37:12.599338Z OPEN -> CLOSED | | +2023-05-25T04:37:12.599352Z | | cluster1-0 left +2023-05-25T04:37:12.600485Z | | cluster1-0 left +2023-05-25T04:37:12.600510Z | | PRIMARY(n=2) +2023-05-25T04:37:12.600902Z | cluster1-0 left | +2023-05-25T04:37:12.600973Z | PRIMARY(n=2) | +2023-05-25T04:37:42.247550Z shutdown complete | | +2023-05-19T03:43:05.371819Z shutdown complete(post.processing.log) | | +2023-05-25T04:38:00.645783Z starting(8.0.31) | | +2023-05-25T04:38:00.659129Z started(cluster) | | +2023-05-25T04:38:00.660295Z recovering gcache | | +2023-05-25T04:38:01.262291Z recovering gcache | | + 10.16.29.241 + (node ip) + V + 10.16.29.34 +2023-05-25T04:38:01.575342Z | | cluster1-0 joined +2023-05-25T04:38:01.575395Z | | cluster1-1 joined +2023-05-25T04:38:01.575465Z | cluster1-0 joined | +2023-05-25T04:38:01.575543Z | cluster1-2 joined | +2023-05-25T04:38:01.575595Z cluster1-2 joined | | +2023-05-25T04:38:01.575616Z cluster1-1 joined | | +2023-05-25T04:38:01.576555Z | | PRIMARY(n=3) +2023-05-25T04:38:01.576715Z | PRIMARY(n=3) | +2023-05-25T04:38:01.860312Z CLOSED -> OPEN | | +2023-05-25T04:38:01.860528Z PRIMARY(n=3) | | +2023-05-25T04:38:01.862258Z OPEN -> PRIMARY | | +2023-05-25T04:38:02.456823Z will receive IST(seqno:5861187) | | +2023-05-25T04:38:02.457884Z | local node will resync cluster1-0 | +2023-05-25T04:38:02.457895Z | | cluster1-1 will resync cluster1-0 +2023-05-25T04:38:02.457924Z | SYNCED -> DONOR | +2023-05-25T04:38:02.457935Z cluster1-1 will resync local node | | +2023-05-25T04:38:02.457961Z PRIMARY -> JOINER | | +2023-05-25T04:38:02.473572Z | IST to cluster1-0(seqno:5861187) | +2023-05-25T04:38:02.904912Z | IST will be used | +2023-05-25T04:38:03.923687Z | | cluster1-1 synced cluster1-0 +2023-05-25T04:38:03.923791Z | finished sending IST to cluster1-0 | +2023-05-25T04:38:03.923865Z | DESYNCED -> JOINED | +2023-05-25T04:38:03.923889Z got IST from cluster1-1 | | +2023-05-25T04:38:03.924496Z | JOINED -> SYNCED | +2023-05-25T04:38:14.821245Z wsrep recovery | | +2023-05-25T04:38:14.952628Z IST received(seqno:5861187) | | +2023-05-25T04:38:14.953589Z JOINER -> JOINED | | +2023-05-25T04:38:14.954030Z JOINED -> SYNCED | | +2023-05-26T03:00:22.581588Z | cluster1-0 joined | +2023-05-26T03:00:22.581645Z | garb joined | +2023-05-26T03:00:22.581658Z | cluster1-2 joined | +2023-05-26T03:00:22.581847Z garb joined | | +2023-05-26T03:00:22.581912Z | | cluster1-0 joined +2023-05-26T03:00:22.581946Z cluster1-2 joined | | +2023-05-26T03:00:22.581970Z cluster1-1 joined | | +2023-05-26T03:00:22.582016Z | | garb joined +2023-05-26T03:00:22.582040Z | | cluster1-1 joined +2023-05-26T03:00:22.583782Z | PRIMARY(n=4) | +2023-05-26T03:00:22.583952Z | | PRIMARY(n=4) +2023-05-26T03:00:22.583976Z PRIMARY(n=4) | | +2023-05-26T03:00:23.082605Z | cluster1-2 will resync garb | +2023-05-26T03:00:23.082696Z | | local node will resync garb +2023-05-26T03:00:23.082706Z | | SYNCED -> DONOR +2023-05-26T03:00:23.082747Z cluster1-2 will resync garb | | +2023-05-26T03:00:34.572248Z | | SST to garb +2023-05-26T03:40:23.808519Z | | finished sending SST to garb +2023-05-26T03:40:23.808574Z | | DESYNCED -> JOINED +2023-05-26T03:40:23.808600Z cluster1-2 synced garb | | +2023-05-26T03:40:23.808954Z | cluster1-2 synced garb | +2023-05-26T03:40:23.809230Z | | JOINED -> SYNCED +2023-05-26T03:40:26.047546Z | | cluster1-0 joined +2023-05-26T03:40:26.047565Z cluster1-2 joined | | +2023-05-26T03:40:26.047613Z | | cluster1-1 joined +2023-05-26T03:40:26.047651Z cluster1-1 joined | | +2023-05-26T03:40:26.047664Z | | garb left +2023-05-26T03:40:26.047712Z garb left | | +2023-05-26T03:40:26.047973Z | cluster1-0 joined | +2023-05-26T03:40:26.048058Z | cluster1-2 joined | +2023-05-26T03:40:26.048117Z | garb left | +2023-05-26T03:40:26.049387Z garb left | | +2023-05-26T03:40:26.049412Z | | garb left +2023-05-26T03:40:26.049454Z PRIMARY(n=3) | | +2023-05-26T03:40:26.049477Z | | PRIMARY(n=3) +2023-05-26T03:40:26.049765Z | garb left | +2023-05-26T03:40:26.049947Z | PRIMARY(n=3) | +2023-05-27T03:00:19.537730Z | cluster1-0 joined | +2023-05-27T03:00:19.537767Z | cluster1-2 joined | +2023-05-27T03:00:19.537775Z | garb joined | +2023-05-27T03:00:19.537993Z cluster1-2 joined | | +2023-05-27T03:00:19.538086Z garb joined | | +2023-05-27T03:00:19.538110Z cluster1-1 joined | | +2023-05-27T03:00:19.539621Z | PRIMARY(n=4) | +2023-05-27T03:00:19.539965Z PRIMARY(n=4) | | +2023-05-27T03:00:19.542417Z | | cluster1-0 joined +2023-05-27T03:00:19.542503Z | | garb joined +2023-05-27T03:00:19.542526Z | | cluster1-1 joined +2023-05-27T03:00:19.544132Z | | PRIMARY(n=4) +2023-05-27T03:00:20.038432Z | cluster1-2 will resync garb | +2023-05-27T03:00:20.038598Z cluster1-2 will resync garb | | +2023-05-27T03:00:20.043068Z | | local node will resync garb +2023-05-27T03:00:20.043091Z | | SYNCED -> DONOR +2023-05-27T03:00:31.529154Z | | SST to garb +2023-05-27T03:39:48.765020Z cluster1-2 synced garb | | +2023-05-27T03:39:48.765166Z | cluster1-2 synced garb | +2023-05-27T03:39:48.768602Z | | finished sending SST to garb +2023-05-27T03:39:48.768676Z | | DESYNCED -> JOINED +2023-05-27T03:39:48.769112Z | | JOINED -> SYNCED +2023-05-27T03:39:51.224587Z cluster1-2 joined | | +2023-05-27T03:39:51.224622Z cluster1-1 joined | | +2023-05-27T03:39:51.224631Z garb left | | +2023-05-27T03:39:51.224821Z | cluster1-0 joined | +2023-05-27T03:39:51.224991Z | cluster1-2 joined | +2023-05-27T03:39:51.225009Z | garb left | +2023-05-27T03:39:51.226003Z garb left | | +2023-05-27T03:39:51.226069Z | garb left | +2023-05-27T03:39:51.226111Z | PRIMARY(n=3) | +2023-05-27T03:39:51.226160Z PRIMARY(n=3) | | +2023-05-27T03:39:51.228020Z | | cluster1-0 joined +2023-05-27T03:39:51.228100Z | | cluster1-1 joined +2023-05-27T03:39:51.228115Z | | garb left +2023-05-27T03:39:51.229544Z | | garb left +2023-05-27T03:39:51.229698Z | | PRIMARY(n=3) +2023-05-28T03:00:20.179665Z cluster1-2 joined | | +2023-05-28T03:00:20.179723Z cluster1-1 joined | | +2023-05-28T03:00:20.179744Z garb joined | | +2023-05-28T03:00:20.180516Z | cluster1-0 joined | +2023-05-28T03:00:20.180583Z | cluster1-2 joined | +2023-05-28T03:00:20.180597Z | garb joined | +2023-05-28T03:00:20.180757Z | | cluster1-0 joined +2023-05-28T03:00:20.180839Z | | cluster1-1 joined +2023-05-28T03:00:20.180863Z | | garb joined +2023-05-28T03:00:20.181677Z PRIMARY(n=4) | | +2023-05-28T03:00:20.182305Z | PRIMARY(n=4) | +2023-05-28T03:00:20.182540Z | | PRIMARY(n=4) +2023-05-28T03:00:20.680781Z cluster1-2 will resync garb | | +2023-05-28T03:00:20.681257Z | cluster1-2 will resync garb | +2023-05-28T03:00:20.681447Z | | local node will resync garb +2023-05-28T03:00:20.681471Z | | SYNCED -> DONOR +2023-05-28T03:00:32.184853Z | | SST to garb +2023-05-28T03:40:51.321660Z | | finished sending SST to garb +2023-05-28T03:40:51.321733Z | | DESYNCED -> JOINED +2023-05-28T03:40:51.321783Z cluster1-2 synced garb | | +2023-05-28T03:40:51.322103Z | cluster1-2 synced garb | +2023-05-28T03:40:51.322593Z | | JOINED -> SYNCED +2023-05-28T03:40:53.656940Z | | cluster1-0 joined +2023-05-28T03:40:53.656966Z | | cluster1-1 joined +2023-05-28T03:40:53.656981Z | | garb left +2023-05-28T03:40:53.657404Z | cluster1-0 joined | +2023-05-28T03:40:53.657428Z | cluster1-2 joined | +2023-05-28T03:40:53.657443Z | garb left | +2023-05-28T03:40:53.657819Z cluster1-2 joined | | +2023-05-28T03:40:53.657840Z cluster1-1 joined | | +2023-05-28T03:40:53.657847Z garb left | | +2023-05-28T03:40:53.658835Z garb left | | +2023-05-28T03:40:53.658995Z | | garb left +2023-05-28T03:40:53.659027Z PRIMARY(n=3) | | +2023-05-28T03:40:53.659156Z | | PRIMARY(n=3) +2023-05-28T03:40:53.659348Z | garb left | +2023-05-28T03:40:53.659384Z | PRIMARY(n=3) | +2023-05-28T08:09:01.368781Z | received shutdown | +2023-05-28T08:19:06.122140Z cluster1-1 suspected to be down | | +2023-05-28T08:19:06.623762Z | | cluster1-1 suspected to be down +2023-05-28T08:19:07.123055Z cluster1-2 joined | | +2023-05-28T08:19:07.124213Z | | cluster1-0 joined +2023-05-28T08:19:07.124217Z cluster1-1 left | | +2023-05-28T08:19:07.124375Z PRIMARY(n=2) | | +2023-05-28T08:19:07.126252Z | | cluster1-1 left +2023-05-28T08:19:07.126298Z | | PRIMARY(n=2) +2023-05-28T08:19:39.773671Z received shutdown | | +2023-05-28T08:23:11.393160Z | | cluster1-0 left +2023-05-28T08:23:11.394190Z NON-PRIMARY(n=1) | | +2023-05-28T08:23:11.394402Z SYNCED -> OPEN | | +2023-05-28T08:23:11.394509Z OPEN -> CLOSED | | +2023-05-28T08:23:11.394531Z | | cluster1-0 left +2023-05-28T08:23:11.394588Z | | PRIMARY(n=1) +2023-05-28T08:19:21.529819Z | starting(8.0.31)(recovery.log) | +2023-05-28T08:19:35.008610Z | wsrep recovery(recovery.log) | +2023-05-28T08:19:45.378813Z | shutdown complete(recovery.log) | +2023-05-28T08:23:23.908187Z | starting(8.0.31) | +2023-05-28T08:23:23.919124Z | started(cluster) | +2023-05-28T08:23:23.920390Z | recovering gcache | +2023-05-28T08:23:24.582927Z | recovering gcache | +2023-05-28T08:23:25.195411Z | CLOSED -> OPEN | +2023-05-28T08:23:25.197121Z | | cluster1-1 joined +2023-05-28T08:23:25.198245Z | cluster1-2 joined | +2023-05-28T08:23:25.200143Z | PRIMARY(n=2) | +2023-05-28T08:23:25.200241Z | | PRIMARY(n=2) +2023-05-28T08:23:25.200793Z | OPEN -> PRIMARY | +2023-05-28T08:23:25.809238Z | will receive IST(seqno:8605683) | +2023-05-28T08:23:25.809921Z | cluster1-2 will resync local node | +2023-05-28T08:23:25.810003Z | PRIMARY -> JOINER | +2023-05-28T08:23:25.810849Z | | local node will resync cluster1-1 +2023-05-28T08:23:25.810936Z | | SYNCED -> DONOR +2023-05-28T08:23:25.824732Z | | IST to cluster1-1(seqno:8605683) +2023-05-28T08:23:26.251768Z | | IST will be used +2023-05-28T08:23:27.268102Z | got IST from cluster1-2 | +2023-05-28T08:23:27.268845Z | | finished sending IST to cluster1-1 +2023-05-28T08:23:27.268914Z | | DESYNCED -> JOINED +2023-05-28T08:23:27.269439Z | | JOINED -> SYNCED +2023-05-28T08:23:39.881488Z | wsrep recovery | +2023-05-28T08:23:40.876116Z | IST received(seqno:8605683) | +2023-05-28T08:23:40.876609Z | JOINER -> JOINED | +2023-05-28T08:23:41.181383Z | JOINED -> SYNCED | +2023-05-28T08:23:48.440344Z shutdown complete | | +2023-05-28T08:24:03.789253Z starting(8.0.31) | | +2023-05-28T08:24:03.802597Z started(cluster) | | +2023-05-28T08:24:03.803745Z recovering gcache | | +2023-05-28T08:24:04.451214Z recovering gcache | | +2023-05-28T08:24:04.580181Z cluster1-2 joined | | +2023-05-28T08:24:04.580186Z cluster1-1 joined | | +2023-05-28T08:24:04.580353Z | cluster1-0 joined | +2023-05-28T08:24:04.580368Z | cluster1-2 joined | +2023-05-28T08:24:04.581094Z | | cluster1-0 joined +2023-05-28T08:24:04.581117Z | | cluster1-1 joined +2023-05-28T08:24:05.582656Z | PRIMARY(n=3) | +2023-05-28T08:24:05.583994Z | | PRIMARY(n=3) +2023-05-28T08:24:06.072396Z CLOSED -> OPEN | | +2023-05-28T08:24:06.072665Z PRIMARY(n=3) | | +2023-05-28T08:24:06.074275Z OPEN -> PRIMARY | | +2023-05-28T08:24:06.692619Z will receive IST(seqno:8607495) | | +2023-05-28T08:24:06.693734Z cluster1-1 will resync local node | | +2023-05-28T08:24:06.693814Z PRIMARY -> JOINER | | +2023-05-28T08:24:06.693995Z | local node will resync cluster1-0 | +2023-05-28T08:24:06.694081Z | SYNCED -> DONOR | +2023-05-28T08:24:06.694718Z | | cluster1-1 will resync cluster1-0 +2023-05-28T08:24:06.710536Z | IST to cluster1-0(seqno:8607495) | +2023-05-28T08:24:07.166023Z | IST will be used | +2023-05-28T08:24:08.182490Z got IST from cluster1-1 | | +2023-05-28T08:24:08.182724Z | finished sending IST to cluster1-0 | +2023-05-28T08:24:08.182760Z | DESYNCED -> JOINED | +2023-05-28T08:24:08.183221Z | JOINED -> SYNCED | +2023-05-28T08:24:08.183529Z | | cluster1-1 synced cluster1-0 +2023-05-28T08:24:19.334302Z wsrep recovery | | +2023-05-28T08:24:20.501118Z IST received(seqno:8607495) | | +2023-05-28T08:24:20.502038Z JOINER -> JOINED | | +2023-05-28T08:24:20.658917Z JOINED -> SYNCED | | +2023-05-28T08:41:19.817710Z received shutdown | | +2023-05-28T08:45:00.599029Z NON-PRIMARY(n=1) | | +2023-05-28T08:45:00.599104Z SYNCED -> OPEN | | +2023-05-28T08:45:00.599137Z OPEN -> CLOSED | | +2023-05-28T08:45:00.599496Z | cluster1-2 joined | +2023-05-28T08:45:00.599569Z | cluster1-0 left | +2023-05-28T08:45:00.600242Z | | cluster1-1 joined +2023-05-28T08:45:00.600380Z | | cluster1-0 left +2023-05-28T08:45:00.601727Z | cluster1-0 left | +2023-05-28T08:45:00.601802Z | PRIMARY(n=2) | +2023-05-28T08:45:00.601872Z | | cluster1-0 left +2023-05-28T08:45:00.602020Z | | PRIMARY(n=2) +2023-05-28T08:45:29.372593Z shutdown complete | | +2023-05-28T08:45:43.616580Z starting(8.0.31) | | +2023-05-28T08:45:43.633451Z started(cluster) | | +2023-05-28T08:45:43.634696Z recovering gcache | | +2023-05-28T08:45:44.344424Z recovering gcache | | +2023-05-28T08:45:44.504384Z cluster1-2 joined | | +2023-05-28T08:45:44.504394Z cluster1-1 joined | | +2023-05-28T08:45:44.504948Z | cluster1-0 joined | +2023-05-28T08:45:44.504992Z | cluster1-2 joined | +2023-05-28T08:45:44.505616Z | | cluster1-0 joined +2023-05-28T08:45:44.505646Z | | cluster1-1 joined +2023-05-28T08:45:44.505932Z | PRIMARY(n=3) | +2023-05-28T08:45:44.506535Z | | PRIMARY(n=3) +2023-05-28T08:45:44.945587Z CLOSED -> OPEN | | +2023-05-28T08:45:44.945768Z PRIMARY(n=3) | | +2023-05-28T08:45:44.947395Z OPEN -> PRIMARY | | +2023-05-28T08:45:45.576073Z will receive IST(seqno:8616014) | | +2023-05-28T08:45:45.577425Z cluster1-1 will resync local node | | +2023-05-28T08:45:45.577501Z PRIMARY -> JOINER | | +2023-05-28T08:45:45.577952Z | local node will resync cluster1-0 | +2023-05-28T08:45:45.578054Z | SYNCED -> DONOR | +2023-05-28T08:45:45.578639Z | | cluster1-1 will resync cluster1-0 +2023-05-28T08:45:45.591166Z | IST to cluster1-0(seqno:8616014) | +2023-05-28T08:45:45.986742Z | IST will be used | +2023-05-28T08:45:47.004479Z got IST from cluster1-1 | | +2023-05-28T08:45:47.005056Z | finished sending IST to cluster1-0 | +2023-05-28T08:45:47.005159Z | DESYNCED -> JOINED | +2023-05-28T08:45:47.005643Z | JOINED -> SYNCED | +2023-05-28T08:45:47.005725Z | | cluster1-1 synced cluster1-0 +2023-05-28T08:45:58.262507Z wsrep recovery | | +2023-05-28T08:45:58.828269Z IST received(seqno:8616014) | | +2023-05-28T08:45:58.829309Z JOINER -> JOINED | | +2023-05-28T08:45:58.829961Z JOINED -> SYNCED | | +2023-05-28T08:52:18.119268Z (repeated x408)too many connections | | +2023-05-28T08:52:47.465770Z | | (repeated x9)too many connections +2023-05-28T08:53:48.040160Z too many connections | | +2023-05-28T08:53:48.056529Z | | too many connections +2023-05-28T08:54:21.486294Z | received shutdown | +2023-05-28T08:55:07.630416Z cluster1-2 joined | | +2023-05-28T08:55:07.630563Z cluster1-1 left | | +2023-05-28T08:55:07.630879Z | NON-PRIMARY(n=1) | +2023-05-28T08:55:07.631003Z | SYNCED -> OPEN | +2023-05-28T08:55:07.631060Z | OPEN -> CLOSED | +2023-05-28T08:55:07.631347Z | | cluster1-0 joined +2023-05-28T08:55:07.631488Z | | cluster1-1 left +2023-05-28T08:55:07.631670Z cluster1-1 left | | +2023-05-28T08:55:07.631751Z PRIMARY(n=2) | | +2023-05-28T08:55:07.632948Z | | cluster1-1 left +2023-05-28T08:55:07.633020Z | | PRIMARY(n=2) +2023-05-28T08:55:37.412783Z | shutdown complete | +2023-05-28T08:55:51.876757Z | starting(8.0.31) | +2023-05-28T08:55:51.886863Z | started(cluster) | +2023-05-28T08:55:51.888167Z | recovering gcache | +2023-05-28T08:55:52.546135Z | recovering gcache | +2023-05-28T08:55:53.692114Z cluster1-1 joined | | +2023-05-28T08:55:53.692201Z cluster1-2 joined | | +2023-05-28T08:55:53.692388Z | cluster1-0 joined | +2023-05-28T08:55:53.692399Z | cluster1-2 joined | +2023-05-28T08:55:53.692911Z | | cluster1-0 joined +2023-05-28T08:55:53.693001Z | | cluster1-1 joined +2023-05-28T08:55:53.694007Z PRIMARY(n=3) | | +2023-05-28T08:55:53.694614Z | | PRIMARY(n=3) +2023-05-28T08:55:54.183486Z | CLOSED -> OPEN | +2023-05-28T08:55:54.183677Z | PRIMARY(n=3) | +2023-05-28T08:55:54.186317Z | OPEN -> PRIMARY | +2023-05-28T08:55:54.807556Z | will receive IST(seqno:8618666) | +2023-05-28T08:55:54.808270Z cluster1-2 will resync cluster1-1 | | +2023-05-28T08:55:54.808658Z | cluster1-2 will resync local node | +2023-05-28T08:55:54.808706Z | PRIMARY -> JOINER | +2023-05-28T08:55:54.809248Z | | local node will resync cluster1-1 +2023-05-28T08:55:54.809337Z | | SYNCED -> DONOR +2023-05-28T08:55:54.823835Z | | IST to cluster1-1(seqno:8618666) +2023-05-28T08:55:55.177191Z | | IST will be used +2023-05-28T08:55:56.193609Z cluster1-2 synced cluster1-1 | | +2023-05-28T08:55:56.193999Z | got IST from cluster1-2 | +2023-05-28T08:55:56.194486Z | | finished sending IST to cluster1-1 +2023-05-28T08:55:56.194594Z | | DESYNCED -> JOINED +2023-05-28T08:55:56.195234Z | | JOINED -> SYNCED +2023-05-28T08:56:07.958413Z | wsrep recovery | +2023-05-28T08:56:08.256725Z | IST received(seqno:8618666) | +2023-05-28T08:56:08.257640Z | JOINER -> JOINED | +2023-05-28T08:56:08.258254Z | JOINED -> SYNCED | +2023-05-29T03:00:17.916558Z | | garb joined +2023-05-29T03:00:17.916588Z | | cluster1-0 joined +2023-05-29T03:00:17.916596Z | | cluster1-1 joined +2023-05-29T03:00:17.917011Z garb joined | | +2023-05-29T03:00:17.917037Z cluster1-1 joined | | +2023-05-29T03:00:17.917043Z cluster1-2 joined | | +2023-05-29T03:00:17.917285Z | garb joined | +2023-05-29T03:00:17.917372Z | cluster1-0 joined | +2023-05-29T03:00:17.917383Z | cluster1-2 joined | +2023-05-29T03:00:18.919405Z | | PRIMARY(n=4) +2023-05-29T03:00:18.919902Z PRIMARY(n=4) | | +2023-05-29T03:00:18.919952Z | PRIMARY(n=4) | +2023-05-29T03:00:19.379393Z | | local node will resync garb +2023-05-29T03:00:19.379427Z | | SYNCED -> DONOR +2023-05-29T03:00:19.379660Z cluster1-2 will resync garb | | +2023-05-29T03:00:19.379736Z | cluster1-2 will resync garb | +2023-05-29T03:00:30.869972Z | | SST to garb +2023-05-29T03:41:39.124527Z | | finished sending SST to garb +2023-05-29T03:41:39.124632Z | | DESYNCED -> JOINED +2023-05-29T03:41:39.125339Z | | JOINED -> SYNCED +2023-05-29T03:41:39.125763Z | cluster1-2 synced garb | +2023-05-29T03:41:39.125854Z cluster1-2 synced garb | | +2023-05-29T05:00:37.738615Z | | garb joined +2023-05-29T05:00:37.738711Z | | cluster1-0 joined +2023-05-29T05:00:37.738735Z | | cluster1-1 joined +2023-05-29T05:00:37.738755Z | | garb joined +2023-05-29T05:00:37.740237Z garb joined | | +2023-05-29T05:00:37.740269Z | garb joined | +2023-05-29T05:00:37.740320Z cluster1-1 joined | | +2023-05-29T05:00:37.740345Z cluster1-2 joined | | +2023-05-29T05:00:37.740366Z garb joined | | +2023-05-29T05:00:37.740402Z | cluster1-0 joined | +2023-05-29T05:00:37.740430Z | cluster1-2 joined | +2023-05-29T05:00:37.740475Z | garb joined | +2023-05-29T05:00:37.740651Z | | PRIMARY(n=5) +2023-05-29T05:00:37.742138Z | PRIMARY(n=5) | +2023-05-29T05:00:37.742177Z PRIMARY(n=5) | | +2023-05-29T05:00:38.239851Z | | local node will resync garb +2023-05-29T05:00:38.239874Z | | SYNCED -> DONOR +2023-05-29T05:00:38.241430Z | cluster1-2 will resync garb | +2023-05-29T05:00:38.241476Z cluster1-2 will resync garb | | +2023-05-29T05:00:49.731756Z | | SST to garb +2023-05-29T05:54:56.049218Z | | finished sending SST to garb +2023-05-29T05:54:56.049291Z | | DESYNCED -> JOINED +2023-05-29T05:54:56.049762Z | | JOINED -> SYNCED +2023-05-29T05:54:56.050912Z | cluster1-2 synced garb | +2023-05-29T05:54:56.050919Z cluster1-2 synced garb | | +2023-05-29T05:54:58.219414Z | | garb joined +2023-05-29T05:54:58.219474Z | | cluster1-0 joined +2023-05-29T05:54:58.219489Z | | cluster1-1 joined +2023-05-29T05:54:58.219504Z | | garb left +2023-05-29T05:54:58.220860Z garb joined | | +2023-05-29T05:54:58.220913Z cluster1-1 joined | | +2023-05-29T05:54:58.220925Z cluster1-2 joined | | +2023-05-29T05:54:58.220954Z garb left | | +2023-05-29T05:54:58.221144Z | garb joined | +2023-05-29T05:54:58.221241Z | cluster1-0 joined | +2023-05-29T05:54:58.221263Z | cluster1-2 joined | +2023-05-29T05:54:58.221280Z | garb left | +2023-05-29T05:55:03.220834Z | | garb suspected to be down +2023-05-29T05:55:03.222309Z garb suspected to be down | | +2023-05-29T05:55:03.722761Z | garb suspected to be down | +2023-05-29T05:55:04.222613Z | | cluster1-0 joined +2023-05-29T05:55:04.222670Z | | cluster1-1 joined +2023-05-29T05:55:04.223778Z | | garb left +2023-05-29T05:55:04.223869Z | | PRIMARY(n=3) +2023-05-29T05:55:04.224239Z | cluster1-0 joined | +2023-05-29T05:55:04.224260Z | cluster1-2 joined | +2023-05-29T05:55:04.224438Z cluster1-1 joined | | +2023-05-29T05:55:04.224539Z cluster1-2 joined | | +2023-05-29T05:55:04.225735Z | garb left | +2023-05-29T05:55:04.225794Z garb left | | +2023-05-29T05:55:04.225815Z | PRIMARY(n=3) | +2023-05-29T05:55:04.225876Z PRIMARY(n=3) | | +2023-05-29T06:18:41.339691Z | | cluster1-0 joined +2023-05-29T06:18:41.339777Z | | cluster1-1 joined +2023-05-29T06:18:41.339802Z | | garb joined +2023-05-29T06:18:41.340638Z cluster1-1 joined | | +2023-05-29T06:18:41.340748Z cluster1-2 joined | | +2023-05-29T06:18:41.340777Z garb joined | | +2023-05-29T06:18:41.340846Z | cluster1-0 joined | +2023-05-29T06:18:41.340903Z | cluster1-2 joined | +2023-05-29T06:18:41.340914Z | garb joined | +2023-05-29T06:18:41.341398Z | | PRIMARY(n=4) +2023-05-29T06:18:41.342417Z | PRIMARY(n=4) | +2023-05-29T06:18:41.342618Z PRIMARY(n=4) | | +2023-05-29T06:18:41.840368Z | | local node will resync garb +2023-05-29T06:18:41.840393Z | | SYNCED -> DONOR +2023-05-29T06:18:41.841468Z | cluster1-2 will resync garb | +2023-05-29T06:18:41.841677Z cluster1-2 will resync garb | | +2023-05-29T06:18:53.370532Z | | SST to garb +2023-05-29T06:21:16.932469Z | | garb joined +2023-05-29T06:21:16.932542Z | | cluster1-0 joined +2023-05-29T06:21:16.932559Z | | cluster1-1 joined +2023-05-29T06:21:16.932574Z | | garb joined +2023-05-29T06:21:16.933460Z | garb joined | +2023-05-29T06:21:16.933530Z | cluster1-0 joined | +2023-05-29T06:21:16.933543Z | cluster1-2 joined | +2023-05-29T06:21:16.933554Z | garb joined | +2023-05-29T06:21:16.933921Z garb joined | | +2023-05-29T06:21:16.933996Z cluster1-1 joined | | +2023-05-29T06:21:16.934011Z cluster1-2 joined | | +2023-05-29T06:21:16.934022Z garb joined | | +2023-05-29T06:21:16.934118Z | | PRIMARY(n=5) +2023-05-29T06:21:16.935175Z | PRIMARY(n=5) | +2023-05-29T06:21:16.935415Z PRIMARY(n=5) | | +2023-05-29T06:21:17.398796Z | | cluster1-1 will resync garb +2023-05-29T06:21:17.399780Z | local node will resync garb | +2023-05-29T06:21:17.399791Z | SYNCED -> DONOR | +2023-05-29T06:21:17.400070Z cluster1-1 will resync garb | | +2023-05-29T06:21:28.872272Z | SST to garb | +2023-05-29T07:16:31.663110Z | cluster1-2 synced garb | +2023-05-29T07:16:31.663588Z cluster1-2 synced garb | | +2023-05-29T07:16:31.664728Z | | finished sending SST to garb +2023-05-29T07:16:31.664811Z | | DESYNCED -> JOINED +2023-05-29T07:16:31.665547Z | | JOINED -> SYNCED +2023-05-29T07:16:33.678136Z | | garb joined +2023-05-29T07:16:33.678217Z | | cluster1-0 joined +2023-05-29T07:16:33.678236Z | | cluster1-1 joined +2023-05-29T07:16:33.678252Z | | garb left +2023-05-29T07:16:34.694046Z | SST error | +2023-05-29T07:16:38.678115Z garb suspected to be down | | +2023-05-29T07:16:38.678140Z cluster1-2 suspected to be down | | +2023-05-29T07:16:38.678174Z | garb suspected to be down | +2023-05-29T07:16:38.678187Z garb suspected to be down | | +2023-05-29T07:16:38.678199Z | cluster1-2 suspected to be down | +2023-05-29T07:16:38.678232Z | garb suspected to be down | +2023-05-29T07:16:38.679876Z | | garb suspected to be down +2023-05-29T07:16:38.679923Z | | cluster1-0 suspected to be down +2023-05-29T07:16:38.679965Z | | cluster1-1 suspected to be down +2023-05-29T07:16:39.178309Z garb suspected to be down | | +2023-05-29T07:16:39.178383Z | garb suspected to be down | +2023-05-29T07:16:39.178407Z cluster1-2 suspected to be down | | +2023-05-29T07:16:39.178437Z garb suspected to be down | | +2023-05-29T07:16:39.178468Z | cluster1-2 suspected to be down | +2023-05-29T07:16:39.178496Z | garb suspected to be down | +2023-05-29T07:16:39.678585Z | garb suspected to be down | +2023-05-29T07:16:39.678620Z garb suspected to be down | | +2023-05-29T07:16:39.678649Z | cluster1-2 suspected to be down | +2023-05-29T07:16:39.678672Z | garb suspected to be down | +2023-05-29T07:16:39.678717Z cluster1-2 suspected to be down | | +2023-05-29T07:16:39.678763Z garb suspected to be down | | +2023-05-29T07:16:39.680986Z | | NON-PRIMARY(n=1) +2023-05-29T07:16:39.681204Z | | SYNCED -> OPEN +2023-05-29T07:16:39.681241Z | | NON-PRIMARY(n=1) +2023-05-29T07:16:40.178884Z garb suspected to be down | | +2023-05-29T07:16:40.178983Z cluster1-2 suspected to be down | | +2023-05-29T07:16:40.179015Z garb suspected to be down | | +2023-05-29T07:16:40.179037Z | garb suspected to be down | +2023-05-29T07:16:40.179126Z | cluster1-2 suspected to be down | +2023-05-29T07:16:40.179154Z | garb suspected to be down | +2023-05-29T07:16:40.679187Z garb suspected to be down | | +2023-05-29T07:16:40.679234Z | garb suspected to be down | +2023-05-29T07:16:40.679286Z | cluster1-2 suspected to be down | +2023-05-29T07:16:40.679287Z cluster1-2 suspected to be down | | +2023-05-29T07:16:40.679299Z | garb suspected to be down | +2023-05-29T07:16:40.679313Z garb suspected to be down | | +2023-05-29T07:16:41.679431Z garb suspected to be down | | +2023-05-29T07:16:41.679447Z cluster1-2 suspected to be down | | +2023-05-29T07:16:41.679615Z | garb suspected to be down | +2023-05-29T07:16:41.679645Z | cluster1-2 suspected to be down | +2023-05-29T07:16:42.179558Z garb suspected to be down | | +2023-05-29T07:16:42.179662Z cluster1-2 suspected to be down | | +2023-05-29T07:16:42.179845Z | garb suspected to be down | +2023-05-29T07:16:42.179935Z | cluster1-2 suspected to be down | +2023-05-29T07:16:42.679809Z garb suspected to be down | | +2023-05-29T07:16:42.679917Z cluster1-2 suspected to be down | | +2023-05-29T07:16:42.680147Z | garb suspected to be down | +2023-05-29T07:16:42.680227Z | cluster1-2 suspected to be down | +2023-05-29T07:16:43.179985Z garb suspected to be down | | +2023-05-29T07:16:43.180032Z cluster1-2 suspected to be down | | +2023-05-29T07:16:43.180363Z | garb suspected to be down | +2023-05-29T07:16:43.180449Z | cluster1-2 suspected to be down | +2023-05-29T07:16:43.680203Z garb suspected to be down | | +2023-05-29T07:16:43.680308Z cluster1-2 suspected to be down | | +2023-05-29T07:16:43.680663Z | garb suspected to be down | +2023-05-29T07:16:43.680746Z | cluster1-2 suspected to be down | +2023-05-29T07:16:44.180424Z garb suspected to be down | | +2023-05-29T07:16:44.180519Z cluster1-2 suspected to be down | | +2023-05-29T07:16:44.181005Z | garb suspected to be down | +2023-05-29T07:16:44.181104Z | cluster1-2 suspected to be down | +2023-05-29T07:16:44.680653Z garb suspected to be down | | +2023-05-29T07:16:44.680716Z cluster1-2 suspected to be down | | +2023-05-29T07:16:44.681319Z | garb suspected to be down | +2023-05-29T07:16:44.681413Z | cluster1-2 suspected to be down | +2023-05-29T07:16:45.180784Z garb suspected to be down | | +2023-05-29T07:16:45.180828Z cluster1-2 suspected to be down | | +2023-05-29T07:16:45.181934Z | garb suspected to be down | +2023-05-29T07:16:45.182024Z | cluster1-2 suspected to be down | +2023-05-29T07:16:45.680937Z garb suspected to be down | | +2023-05-29T07:16:45.680994Z cluster1-2 suspected to be down | | +2023-05-29T07:16:45.682205Z | garb suspected to be down | +2023-05-29T07:16:45.682266Z | cluster1-2 suspected to be down | +2023-05-29T07:16:46.181058Z garb suspected to be down | | +2023-05-29T07:16:46.181106Z cluster1-2 suspected to be down | | +2023-05-29T07:16:46.182370Z | garb suspected to be down | +2023-05-29T07:16:46.182475Z | cluster1-2 suspected to be down | +2023-05-29T07:16:46.681295Z garb suspected to be down | | +2023-05-29T07:16:46.681392Z cluster1-2 suspected to be down | | +2023-05-29T07:16:46.682637Z | garb suspected to be down | +2023-05-29T07:16:46.682671Z | cluster1-2 suspected to be down | +2023-05-29T07:16:47.181556Z garb suspected to be down | | +2023-05-29T07:16:47.181655Z cluster1-2 suspected to be down | | +2023-05-29T07:16:47.182793Z | garb suspected to be down | +2023-05-29T07:16:47.182876Z | cluster1-2 suspected to be down | +2023-05-29T07:16:47.681851Z garb suspected to be down | | +2023-05-29T07:16:47.681934Z cluster1-2 suspected to be down | | +2023-05-29T07:16:47.683061Z | garb suspected to be down | +2023-05-29T07:16:47.683101Z | cluster1-2 suspected to be down | +2023-05-29T07:16:48.182043Z garb suspected to be down | | +2023-05-29T07:16:48.182140Z cluster1-2 suspected to be down | | +2023-05-29T07:16:48.183482Z | garb suspected to be down | +2023-05-29T07:16:48.183563Z | cluster1-2 suspected to be down | +2023-05-29T07:16:48.682756Z | garb left | +2023-05-29T07:16:48.682898Z | NON-PRIMARY(n=1) | +2023-05-29T07:16:48.682999Z | DESYNCED -> OPEN | +2023-05-29T07:16:48.683014Z | NON-PRIMARY(n=1) | +2023-05-29T07:16:48.683734Z garb left | | +2023-05-29T07:16:48.683956Z NON-PRIMARY(n=1) | | +2023-05-29T07:16:48.684193Z SYNCED -> OPEN | | +2023-05-29T07:16:48.684230Z NON-PRIMARY(n=1) | | +2023-05-29T07:16:49.684876Z | cluster1-0 joined | +2023-05-29T07:16:49.684971Z | cluster1-2 joined | +2023-05-29T07:16:49.685351Z cluster1-1 joined | | +2023-05-29T07:16:49.685402Z cluster1-2 joined | | +2023-05-29T07:16:49.685497Z | NON-PRIMARY(n=3) | +2023-05-29T07:16:49.686149Z NON-PRIMARY(n=3) | | +2023-05-29T07:16:49.686256Z | | unspecified joined +2023-05-29T07:16:49.686310Z | | cluster1-1 joined +2023-05-29T07:16:49.687051Z | | NON-PRIMARY(n=3) +2023-05-29T07:17:06.846743Z | | received shutdown +2023-05-29T07:17:15.812255Z received shutdown | | +2023-05-29T07:17:16.848791Z | unspecified joined | +2023-05-29T07:17:16.848924Z | unspecified left | +2023-05-29T07:17:16.849374Z unspecified joined | | +2023-05-29T07:17:16.849499Z unspecified left | | +2023-05-29T07:17:16.850012Z | NON-PRIMARY(n=2) | +2023-05-29T07:17:16.850477Z | | NON-PRIMARY(n=1) +2023-05-29T07:17:16.850489Z NON-PRIMARY(n=2) | | +2023-05-29T07:17:16.850794Z | | OPEN -> CLOSED +2023-05-29T07:17:17.387918Z | received shutdown | +2023-05-29T07:17:26.815037Z | unspecified left | +2023-05-29T07:17:26.815352Z | NON-PRIMARY(n=1) | +2023-05-29T07:17:26.816357Z NON-PRIMARY(n=1) | | +2023-05-29T07:17:26.816671Z OPEN -> CLOSED | | +2023-05-29T07:17:27.390229Z | OPEN -> CLOSED | +2023-05-29T07:17:45.090041Z | | shutdown complete +2023-05-29T07:17:52.956890Z shutdown complete | | +2023-05-29T07:17:55.556216Z | shutdown complete | +2023-05-29T07:19:34.593510Z starting(8.0.31) | | +2023-05-29T07:19:34.607812Z started(cluster) | | +2023-05-29T07:19:34.608840Z safe_to_bootstrap: 1 | | +2023-05-29T07:19:34.608976Z recovering gcache | | +2023-05-29T07:19:35.197239Z recovering gcache | | +2023-05-29T07:19:35.284187Z bootstrapping | | +2023-05-29T07:19:35.284852Z CLOSED -> OPEN | | +2023-05-29T07:19:35.285076Z PRIMARY(n=1) | | +2023-05-29T07:19:35.286156Z (restored)OPEN -> JOINED | | +2023-05-29T07:19:35.286229Z JOINED -> SYNCED | | +2023-05-29T07:20:12.578000Z | | starting(8.0.31) +2023-05-29T07:20:12.581823Z | | started(cluster) +2023-05-29T07:20:12.583060Z | | recovering gcache +2023-05-29T07:20:13.267704Z | | recovering gcache +2023-05-29T07:20:13.855900Z cluster1-2 joined | | +2023-05-29T07:20:13.857279Z | | cluster1-0 joined +2023-05-29T07:20:13.857329Z PRIMARY(n=2) | | +2023-05-29T07:20:14.356588Z | | CLOSED -> OPEN +2023-05-29T07:20:14.356749Z | | PRIMARY(n=2) +2023-05-29T07:20:14.358922Z | | OPEN -> PRIMARY +2023-05-29T07:20:14.962538Z local node will resync cluster1-2 | | +2023-05-29T07:20:14.962618Z SYNCED -> DONOR | | +2023-05-29T07:20:14.962620Z | | will receive IST(seqno:9339113) +2023-05-29T07:20:14.963634Z | | cluster1-0 will resync local node +2023-05-29T07:20:14.963723Z | | PRIMARY -> JOINER +2023-05-29T07:20:14.979257Z IST to cluster1-2(seqno:9339113) | | +2023-05-29T07:20:15.349516Z IST will be used | | +2023-05-29T07:20:16.366198Z finished sending IST to cluster1-2 | | +2023-05-29T07:20:16.366270Z DESYNCED -> JOINED | | +2023-05-29T07:20:16.366599Z JOINED -> SYNCED | | +2023-05-29T07:20:16.367386Z | | got IST from cluster1-0 +2023-05-29T07:20:28.138289Z | | wsrep recovery +2023-05-29T07:20:28.379593Z | | IST received(seqno:9339113) +2023-05-29T07:20:28.380334Z | | JOINER -> JOINED +2023-05-29T07:20:28.455306Z | | JOINED -> SYNCED +2023-05-29T07:20:30.982370Z | starting(8.0.31) | +2023-05-29T07:20:30.994020Z | started(cluster) | +2023-05-29T07:20:30.995328Z | recovering gcache | +2023-05-29T07:20:31.626044Z | recovering gcache | +2023-05-29T07:20:32.215655Z | cluster1-0 joined | +2023-05-29T07:20:32.215681Z | cluster1-2 joined | +2023-05-29T07:20:32.216241Z cluster1-2 joined | | +2023-05-29T07:20:32.216334Z cluster1-1 joined | | +2023-05-29T07:20:32.217405Z | | cluster1-0 joined +2023-05-29T07:20:32.217511Z | | cluster1-1 joined +2023-05-29T07:20:32.217863Z PRIMARY(n=3) | | +2023-05-29T07:20:32.219022Z | | PRIMARY(n=3) +2023-05-29T07:20:32.714734Z | CLOSED -> OPEN | +2023-05-29T07:20:32.714910Z | PRIMARY(n=3) | +2023-05-29T07:20:32.717573Z | OPEN -> PRIMARY | +2023-05-29T07:20:33.299585Z | will receive IST(seqno:9339350) | +2023-05-29T07:20:33.300653Z | cluster1-0 will resync local node | +2023-05-29T07:20:33.300727Z | PRIMARY -> JOINER | +2023-05-29T07:20:33.301361Z local node will resync cluster1-1 | | +2023-05-29T07:20:33.301459Z SYNCED -> DONOR | | +2023-05-29T07:20:33.302423Z | | cluster1-0 will resync cluster1-1 +2023-05-29T07:20:33.314392Z IST to cluster1-1(seqno:9339350) | | +2023-05-29T07:20:33.730384Z IST will be used | | +2023-05-29T07:20:34.750551Z | got IST from cluster1-0 | +2023-05-29T07:20:34.751280Z finished sending IST to cluster1-1 | | +2023-05-29T07:20:34.751368Z DESYNCED -> JOINED | | +2023-05-29T07:20:34.751983Z JOINED -> SYNCED | | +2023-05-29T07:20:34.752408Z | | cluster1-0 synced cluster1-1 +2023-05-29T07:20:47.797067Z | wsrep recovery | +2023-05-29T07:20:48.198421Z | IST received(seqno:9339350) | +2023-05-29T07:20:48.199500Z | JOINER -> JOINED | +2023-05-29T07:20:48.200243Z | JOINED -> SYNCED | +2023-05-29T07:54:32.160958Z | garb joined | +2023-05-29T07:54:32.161002Z | cluster1-0 joined | +2023-05-29T07:54:32.161009Z | cluster1-2 joined | +2023-05-29T07:54:32.161040Z garb joined | | +2023-05-29T07:54:32.161077Z cluster1-2 joined | | +2023-05-29T07:54:32.161084Z cluster1-1 joined | | +2023-05-29T07:54:32.161868Z | PRIMARY(n=4) | +2023-05-29T07:54:32.161979Z PRIMARY(n=4) | | +2023-05-29T07:54:32.164088Z | | garb joined +2023-05-29T07:54:32.164138Z | | cluster1-0 joined +2023-05-29T07:54:32.164147Z | | cluster1-1 joined +2023-05-29T07:54:32.165047Z | | PRIMARY(n=4) +2023-05-29T07:54:32.440770Z | cluster1-2 will resync garb | +2023-05-29T07:54:32.440867Z cluster1-2 will resync garb | | +2023-05-29T07:54:32.443905Z | | local node will resync garb +2023-05-29T07:54:32.443943Z | | SYNCED -> DONOR +2023-05-29T07:54:43.911767Z | | SST to garb +2023-05-29T08:45:19.117463Z | cluster1-2 synced garb | +2023-05-29T08:45:19.117474Z cluster1-2 synced garb | | +2023-05-29T08:45:19.121268Z | | finished sending SST to garb +2023-05-29T08:45:19.121298Z | | DESYNCED -> JOINED +2023-05-29T08:45:19.122002Z | | JOINED -> SYNCED +2023-05-29T08:45:21.448009Z cluster1-2 joined | | +2023-05-29T08:45:21.448042Z cluster1-1 joined | | +2023-05-29T08:45:21.448063Z garb left | | +2023-05-29T08:45:21.448135Z | cluster1-0 joined | +2023-05-29T08:45:21.448173Z | cluster1-2 joined | +2023-05-29T08:45:21.448197Z | garb left | +2023-05-29T08:45:21.449441Z garb left | | +2023-05-29T08:45:21.449521Z PRIMARY(n=3) | | +2023-05-29T08:45:21.449586Z | garb left | +2023-05-29T08:45:21.449684Z | PRIMARY(n=3) | +2023-05-29T08:45:21.451385Z | | cluster1-0 joined +2023-05-29T08:45:21.451433Z | | cluster1-1 joined +2023-05-29T08:45:21.451442Z | | garb left +2023-05-29T08:45:21.454564Z | | garb left +2023-05-29T08:45:21.454608Z | | PRIMARY(n=3) + +identifier cluster1-0 cluster1-1 cluster1-2 +current path tests/logs/operator_ambiguous_ips/node1.log tests/logs/operator_ambiguous_ips/node2.log tests/logs/operator_ambiguous_ips/node3.log +last known ip 10.16.29.34 10.16.27.98 10.16.28.213 +last known name cluster1-0 cluster1-1 cluster1-2 +mysql version 8.0.31 8.0.31 8.0.31 diff --git a/src/go/pt-galera-log-explainer/types/operator.go b/src/go/pt-galera-log-explainer/types/operator.go index af91ade2..506bc43b 100644 --- a/src/go/pt-galera-log-explainer/types/operator.go +++ b/src/go/pt-galera-log-explainer/types/operator.go @@ -1,5 +1,9 @@ package types +const ( + OperatorLogPrefix = `{"log":"` +) + type OperatorMetadata struct { PodName string Deployment string