mirror of
https://github.com/percona/percona-toolkit.git
synced 2025-09-25 13:46:22 +00:00
Add: concurrent SSTs handling
It is a thing: 2 nodes joining at the same time, with 2 JOINERs and 2 DONORs cluster-wide It can happen on operators with 2 garbd joining at the same time Before, pt-galera-log-explainer was using SST metadata naively. Basically if a node was DONOR and we found a "transfer completed" message, we assumed the donor name we found is the correct one. So for concurrent SSTs, donors were swapping names. Now, it is handled by a map, indexed by a donor name. To know if a node is actual donor or not, it now compare timestamps of events. It assumes both "selected donor" and "shifting DONOR" messages should have happen in less than 0.01 secs to avoid any conflict. Regression tests coming in next commit with an operator logs having concurrent SSTs. Another conflicts was sometimes breaking the test depending on the order on which we read files, hence why it's not added here yet
This commit is contained in:
Yoann La Cancellera
@@ -61,7 +61,7 @@ var (
|
||||
regexSeqno = "(?P<" + groupSeqno + ">[0-9]+)"
|
||||
regexNodeIP = "(?P<" + groupNodeIP + ">[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})"
|
||||
regexNodeIPMethod = "(?P<" + groupMethod + ">.+)://" + regexNodeIP + ":[0-9]{1,6}"
|
||||
regexIdx = "(?P<" + groupIdx + ">-?[0-9]{1,2})"
|
||||
regexIdx = "(?P<" + groupIdx + ">-?[0-9]{1,2})(\\.-?[0-9])?"
|
||||
regexVersion = "(?P<" + groupVersion + ">(5|8|10|11)\\.[0-9]\\.[0-9]{1,2})"
|
||||
regexErrorMD5 = "(?P<" + groupErrorMD5 + ">[a-z0-9]*)"
|
||||
)
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"io/ioutil"
|
||||
"os/exec"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
@@ -740,8 +741,8 @@ func TestRegexes(t *testing.T) {
|
||||
|
||||
{
|
||||
log: "2001-01-01T01:01:01.000000Z 0 [Note] WSREP: Member 2.0 (node2) requested state transfer from '*any*'. Selected 0.0 (node1)(SYNCED) as donor.",
|
||||
inputCtx: types.LogCtx{},
|
||||
expectedCtx: types.LogCtx{},
|
||||
inputCtx: types.LogCtx{SSTs: map[string]types.SST{}},
|
||||
expectedCtx: types.LogCtx{SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2", SelectionTimestamp: timeMustParse("2001-01-01T01:01:01.000000Z")}}},
|
||||
expectedOut: "node1 will resync node2",
|
||||
mapToTest: SSTMap,
|
||||
key: "RegexSSTRequestSuccess",
|
||||
@@ -749,8 +750,8 @@ func TestRegexes(t *testing.T) {
|
||||
{
|
||||
name: "with fqdn",
|
||||
log: "2001-01-01T01:01:01.000000Z 0 [Note] [MY-000000] [Galera] Member 2.0 (node2.host.com) requested state transfer from '*any*'. Selected 0.0 (node1.host.com)(SYNCED) as donor.",
|
||||
inputCtx: types.LogCtx{},
|
||||
expectedCtx: types.LogCtx{},
|
||||
inputCtx: types.LogCtx{SSTs: map[string]types.SST{}},
|
||||
expectedCtx: types.LogCtx{SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2", SelectionTimestamp: timeMustParse("2001-01-01T01:01:01.000000Z")}}},
|
||||
expectedOut: "node1 will resync node2",
|
||||
mapToTest: SSTMap,
|
||||
key: "RegexSSTRequestSuccess",
|
||||
@@ -760,10 +761,11 @@ func TestRegexes(t *testing.T) {
|
||||
log: "2001-01-01T01:01:01.000000Z 0 [Note] WSREP: Member 2.0 (node2) requested state transfer from '*any*'. Selected 0.0 (node1)(SYNCED) as donor.",
|
||||
inputCtx: types.LogCtx{
|
||||
OwnNames: []string{"node2"},
|
||||
SSTs: map[string]types.SST{},
|
||||
},
|
||||
expectedCtx: types.LogCtx{
|
||||
OwnNames: []string{"node2"},
|
||||
SST: types.SST{ResyncedFromNode: "node1"},
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2", SelectionTimestamp: timeMustParse("2001-01-01T01:01:01.000000Z")}},
|
||||
},
|
||||
expectedOut: "node1 will resync local node",
|
||||
mapToTest: SSTMap,
|
||||
@@ -774,10 +776,11 @@ func TestRegexes(t *testing.T) {
|
||||
log: "2001-01-01T01:01:01.000000Z 0 [Note] WSREP: Member 2.0 (node2) requested state transfer from '*any*'. Selected 0.0 (node1)(SYNCED) as donor.",
|
||||
inputCtx: types.LogCtx{
|
||||
OwnNames: []string{"node1"},
|
||||
SSTs: map[string]types.SST{},
|
||||
},
|
||||
expectedCtx: types.LogCtx{
|
||||
OwnNames: []string{"node1"},
|
||||
SST: types.SST{ResyncingNode: "node2"},
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2", SelectionTimestamp: timeMustParse("2001-01-01T01:01:01.000000Z")}},
|
||||
},
|
||||
expectedOut: "local node will resync node2",
|
||||
mapToTest: SSTMap,
|
||||
@@ -807,9 +810,13 @@ func TestRegexes(t *testing.T) {
|
||||
},
|
||||
|
||||
{
|
||||
log: "2001-01-01T01:01:01.000000Z 0 [Note] WSREP: 0.0 (node1): State transfer to 2.0 (node2) complete.",
|
||||
inputCtx: types.LogCtx{},
|
||||
expectedCtx: types.LogCtx{},
|
||||
log: "2001-01-01T01:01:01.000000Z 0 [Note] WSREP: 0.0 (node1): State transfer to 2.0 (node2) complete.",
|
||||
inputCtx: types.LogCtx{
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2"}},
|
||||
},
|
||||
expectedCtx: types.LogCtx{
|
||||
SSTs: map[string]types.SST{},
|
||||
},
|
||||
expectedOut: "node1 synced node2",
|
||||
mapToTest: SSTMap,
|
||||
key: "RegexSSTComplete",
|
||||
@@ -819,10 +826,10 @@ func TestRegexes(t *testing.T) {
|
||||
log: "2001-01-01T01:01:01.000000Z 0 [Note] WSREP: 0.0 (node1): State transfer to 2.0 (node2) complete.",
|
||||
inputCtx: types.LogCtx{
|
||||
OwnNames: []string{"node2"},
|
||||
SST: types.SST{ResyncedFromNode: "node1"},
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2"}},
|
||||
},
|
||||
expectedCtx: types.LogCtx{
|
||||
SST: types.SST{ResyncedFromNode: ""},
|
||||
SSTs: map[string]types.SST{},
|
||||
OwnNames: []string{"node2"},
|
||||
},
|
||||
expectedOut: "got SST from node1",
|
||||
@@ -834,10 +841,10 @@ func TestRegexes(t *testing.T) {
|
||||
log: "2001-01-01T01:01:01.000000Z 0 [Note] WSREP: 0.0 (node1): State transfer to 2.0 (node2) complete.",
|
||||
inputCtx: types.LogCtx{
|
||||
OwnNames: []string{"node2"},
|
||||
SST: types.SST{ResyncedFromNode: "node1", Type: "IST"},
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2", Type: "IST"}},
|
||||
},
|
||||
expectedCtx: types.LogCtx{
|
||||
SST: types.SST{ResyncedFromNode: "", Type: ""},
|
||||
SSTs: map[string]types.SST{},
|
||||
OwnNames: []string{"node2"},
|
||||
},
|
||||
expectedOut: "got IST from node1",
|
||||
@@ -849,10 +856,10 @@ func TestRegexes(t *testing.T) {
|
||||
log: "2001-01-01T01:01:01.000000Z 0 [Note] WSREP: 0.0 (node1): State transfer to 2.0 (node2) complete.",
|
||||
inputCtx: types.LogCtx{
|
||||
OwnNames: []string{"node1"},
|
||||
SST: types.SST{ResyncingNode: "node2"},
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2"}},
|
||||
},
|
||||
expectedCtx: types.LogCtx{
|
||||
SST: types.SST{ResyncingNode: ""},
|
||||
SSTs: map[string]types.SST{},
|
||||
OwnNames: []string{"node1"},
|
||||
},
|
||||
expectedOut: "finished sending SST to node2",
|
||||
@@ -864,48 +871,16 @@ func TestRegexes(t *testing.T) {
|
||||
log: "2001-01-01T01:01:01.000000Z 0 [Note] WSREP: 0.0 (node1): State transfer to 2.0 (node2) complete.",
|
||||
inputCtx: types.LogCtx{
|
||||
OwnNames: []string{"node1"},
|
||||
SST: types.SST{ResyncingNode: "node2", Type: "IST"},
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2", Type: "IST"}},
|
||||
},
|
||||
expectedCtx: types.LogCtx{
|
||||
SST: types.SST{ResyncingNode: "", Type: ""},
|
||||
SSTs: map[string]types.SST{},
|
||||
OwnNames: []string{"node1"},
|
||||
},
|
||||
expectedOut: "finished sending IST to node2",
|
||||
mapToTest: SSTMap,
|
||||
key: "RegexSSTComplete",
|
||||
},
|
||||
{
|
||||
name: "with donor name",
|
||||
log: "2001-01-01T01:01:01.000000Z 0 [Note] WSREP: 0.0 (node1): State transfer to 2.0 (node2) complete.",
|
||||
inputCtx: types.LogCtx{
|
||||
SST: types.SST{ResyncingNode: "node2", Type: "IST"},
|
||||
},
|
||||
expectedCtx: types.LogCtx{
|
||||
SST: types.SST{ResyncingNode: "", Type: ""},
|
||||
OwnNames: []string{"node1"},
|
||||
},
|
||||
inputState: "DONOR",
|
||||
expectedState: "DONOR",
|
||||
expectedOut: "finished sending IST to node2",
|
||||
mapToTest: SSTMap,
|
||||
key: "RegexSSTComplete",
|
||||
},
|
||||
{
|
||||
name: "with joiner name",
|
||||
log: "2001-01-01T01:01:01.000000Z 0 [Note] WSREP: 0.0 (node1): State transfer to 2.0 (node2) complete.",
|
||||
inputCtx: types.LogCtx{
|
||||
SST: types.SST{ResyncingNode: "node2", Type: "IST"},
|
||||
},
|
||||
expectedCtx: types.LogCtx{
|
||||
SST: types.SST{ResyncingNode: "", Type: ""},
|
||||
OwnNames: []string{"node2"},
|
||||
},
|
||||
inputState: "JOINER",
|
||||
expectedState: "JOINER",
|
||||
expectedOut: "got IST from node1",
|
||||
mapToTest: SSTMap,
|
||||
key: "RegexSSTComplete",
|
||||
},
|
||||
|
||||
{
|
||||
log: "2001-01-01T01:01:01.000000Z 0 [Note] WSREP: 0.0 (node1): State transfer to -1.-1 (left the group) complete.",
|
||||
@@ -931,8 +906,15 @@ func TestRegexes(t *testing.T) {
|
||||
},
|
||||
|
||||
{
|
||||
log: "2001-01-01T01:01:01.000000Z WSREP_SST: [INFO] Proceeding with SST.........",
|
||||
expectedCtx: types.LogCtx{SST: types.SST{Type: "SST"}},
|
||||
log: "2001-01-01T01:01:01.000000Z WSREP_SST: [INFO] Proceeding with SST.........",
|
||||
inputCtx: types.LogCtx{
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2"}},
|
||||
OwnNames: []string{"node2"},
|
||||
},
|
||||
expectedCtx: types.LogCtx{
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2", Type: "SST"}},
|
||||
OwnNames: []string{"node2"},
|
||||
},
|
||||
expectedState: "JOINER",
|
||||
expectedOut: "receiving SST",
|
||||
mapToTest: SSTMap,
|
||||
@@ -941,7 +923,6 @@ func TestRegexes(t *testing.T) {
|
||||
|
||||
{
|
||||
log: "2001-01-01T01:01:01.000000Z WSREP_SST: [INFO] Streaming the backup to joiner at 172.17.0.2 4444",
|
||||
expectedCtx: types.LogCtx{SST: types.SST{ResyncingNode: "172.17.0.2"}},
|
||||
expectedState: "DONOR",
|
||||
expectedOut: "SST to 172.17.0.2",
|
||||
mapToTest: SSTMap,
|
||||
@@ -957,8 +938,15 @@ func TestRegexes(t *testing.T) {
|
||||
},
|
||||
|
||||
{
|
||||
log: "2001-01-01 1:01:01 140433613571840 [Note] WSREP: async IST sender starting to serve tcp://172.17.0.2:4568 sending 2-116",
|
||||
expectedCtx: types.LogCtx{SST: types.SST{Type: "IST"}},
|
||||
log: "2001-01-01 1:01:01 140433613571840 [Note] WSREP: async IST sender starting to serve tcp://172.17.0.2:4568 sending 2-116",
|
||||
inputCtx: types.LogCtx{
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2"}},
|
||||
OwnNames: []string{"node1"},
|
||||
},
|
||||
expectedCtx: types.LogCtx{
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2", Type: "IST"}},
|
||||
OwnNames: []string{"node1"},
|
||||
},
|
||||
expectedState: "DONOR",
|
||||
expectedOut: "IST to 172.17.0.2(seqno:116)",
|
||||
mapToTest: SSTMap,
|
||||
@@ -966,25 +954,46 @@ func TestRegexes(t *testing.T) {
|
||||
},
|
||||
|
||||
{
|
||||
log: "2001-01-01T01:01:01.000000Z 0 [Note] [MY-000000] [Galera] Prepared IST receiver for 114-116, listening at: ssl://172.17.0.2:4568",
|
||||
expectedCtx: types.LogCtx{SST: types.SST{Type: "IST"}},
|
||||
log: "2001-01-01T01:01:01.000000Z 0 [Note] [MY-000000] [Galera] Prepared IST receiver for 114-116, listening at: ssl://172.17.0.2:4568",
|
||||
inputCtx: types.LogCtx{
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2"}},
|
||||
OwnNames: []string{"node2"},
|
||||
},
|
||||
expectedCtx: types.LogCtx{
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2", Type: "IST"}},
|
||||
OwnNames: []string{"node2"},
|
||||
},
|
||||
expectedState: "JOINER",
|
||||
expectedOut: "will receive IST(seqno:116)",
|
||||
mapToTest: SSTMap,
|
||||
key: "RegexISTReceiver",
|
||||
},
|
||||
{
|
||||
log: "2001-01-01T01:01:01.000000Z 0 [Note] [MY-000000] [Galera] Prepared IST receiver for 0-116, listening at: ssl://172.17.0.2:4568",
|
||||
expectedCtx: types.LogCtx{SST: types.SST{Type: "SST"}},
|
||||
log: "2001-01-01T01:01:01.000000Z 0 [Note] [MY-000000] [Galera] Prepared IST receiver for 0-116, listening at: ssl://172.17.0.2:4568",
|
||||
inputCtx: types.LogCtx{
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2"}},
|
||||
OwnNames: []string{"node2"},
|
||||
},
|
||||
expectedCtx: types.LogCtx{
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2", Type: "SST"}},
|
||||
OwnNames: []string{"node2"},
|
||||
},
|
||||
expectedState: "JOINER",
|
||||
expectedOut: "will receive SST",
|
||||
mapToTest: SSTMap,
|
||||
key: "RegexISTReceiver",
|
||||
},
|
||||
{
|
||||
name: "mdb variant",
|
||||
log: "2001-01-01T01:01:01.000000Z 0 [Note] WSREP: Prepared IST receiver, listening at: ssl://172.17.0.2:4568",
|
||||
expectedCtx: types.LogCtx{SST: types.SST{Type: "IST"}},
|
||||
name: "mdb variant",
|
||||
log: "2001-01-01T01:01:01.000000Z 0 [Note] WSREP: Prepared IST receiver, listening at: ssl://172.17.0.2:4568",
|
||||
inputCtx: types.LogCtx{
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2"}},
|
||||
OwnNames: []string{"node2"},
|
||||
},
|
||||
expectedCtx: types.LogCtx{
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2", Type: "IST"}},
|
||||
OwnNames: []string{"node2"},
|
||||
},
|
||||
expectedState: "JOINER",
|
||||
expectedOut: "will receive IST",
|
||||
mapToTest: SSTMap,
|
||||
@@ -1012,26 +1021,70 @@ func TestRegexes(t *testing.T) {
|
||||
},
|
||||
|
||||
{
|
||||
log: "2001-01-01T01:01:01.000000Z 1 [Note] WSREP: Failed to prepare for incremental state transfer: Local state UUID (00000000-0000-0000-0000-000000000000) does not match group state UUID (ed16c932-84b3-11ed-998c-8e3ae5bc328f): 1 (Operation not permitted)",
|
||||
expectedCtx: types.LogCtx{SST: types.SST{Type: "SST"}},
|
||||
expectedOut: "IST is not applicable",
|
||||
mapToTest: SSTMap,
|
||||
key: "RegexFailedToPrepareIST",
|
||||
log: "2001-01-01T01:01:01.000000Z 1 [Note] WSREP: Failed to prepare for incremental state transfer: Local state UUID (00000000-0000-0000-0000-000000000000) does not match group state UUID (ed16c932-84b3-11ed-998c-8e3ae5bc328f): 1 (Operation not permitted)",
|
||||
inputState: "JOINER",
|
||||
inputCtx: types.LogCtx{
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2"}},
|
||||
OwnNames: []string{"node2"},
|
||||
},
|
||||
expectedCtx: types.LogCtx{
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2", Type: "SST"}},
|
||||
OwnNames: []string{"node2"},
|
||||
},
|
||||
expectedState: "JOINER",
|
||||
expectedOut: "IST is not applicable",
|
||||
mapToTest: SSTMap,
|
||||
key: "RegexFailedToPrepareIST",
|
||||
},
|
||||
{
|
||||
log: "2001-01-01T01:01:01.000000Z 1 [Warning] WSREP: Failed to prepare for incremental state transfer: Local state seqno is undefined: 1 (Operation not permitted)",
|
||||
expectedCtx: types.LogCtx{SST: types.SST{Type: "SST"}},
|
||||
expectedOut: "IST is not applicable",
|
||||
mapToTest: SSTMap,
|
||||
key: "RegexFailedToPrepareIST",
|
||||
log: "2001-01-01T01:01:01.000000Z 1 [Warning] WSREP: Failed to prepare for incremental state transfer: Local state seqno is undefined: 1 (Operation not permitted)",
|
||||
inputState: "JOINER",
|
||||
inputCtx: types.LogCtx{
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2"}},
|
||||
OwnNames: []string{"node2"},
|
||||
},
|
||||
expectedCtx: types.LogCtx{
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2", Type: "SST"}},
|
||||
OwnNames: []string{"node2"},
|
||||
},
|
||||
expectedState: "JOINER",
|
||||
expectedOut: "IST is not applicable",
|
||||
mapToTest: SSTMap,
|
||||
key: "RegexFailedToPrepareIST",
|
||||
},
|
||||
|
||||
{
|
||||
log: "2001-01-01T01:01:01.000000Z WSREP_SST: [INFO] Bypassing SST. Can work it through IST",
|
||||
expectedCtx: types.LogCtx{SST: types.SST{Type: "IST"}},
|
||||
expectedOut: "IST will be used",
|
||||
mapToTest: SSTMap,
|
||||
key: "RegexBypassSST",
|
||||
log: "2001-01-01T01:01:01.000000Z WSREP_SST: [INFO] Bypassing SST. Can work it through IST",
|
||||
inputState: "JOINER",
|
||||
inputCtx: types.LogCtx{
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2"}},
|
||||
OwnNames: []string{"node2"},
|
||||
},
|
||||
expectedCtx: types.LogCtx{
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2", Type: "IST"}},
|
||||
OwnNames: []string{"node2"},
|
||||
},
|
||||
expectedState: "JOINER",
|
||||
expectedOut: "IST will be used",
|
||||
mapToTest: SSTMap,
|
||||
key: "RegexBypassSST",
|
||||
},
|
||||
|
||||
{
|
||||
log: "2001-01-01T01:01:01.000000Z 0 [Note] [MY-000000] [WSREP-SST] xtrabackup_ist received from donor: Running IST",
|
||||
inputState: "JOINER",
|
||||
inputCtx: types.LogCtx{
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2"}},
|
||||
OwnNames: []string{"node2"},
|
||||
},
|
||||
expectedCtx: types.LogCtx{
|
||||
SSTs: map[string]types.SST{"node1": types.SST{Donor: "node1", Joiner: "node2", Type: "IST"}},
|
||||
OwnNames: []string{"node2"},
|
||||
},
|
||||
expectedState: "JOINER",
|
||||
expectedOut: "IST running",
|
||||
mapToTest: SSTMap,
|
||||
key: "RegexXtrabackupISTReceived",
|
||||
},
|
||||
|
||||
{
|
||||
@@ -1256,6 +1309,11 @@ func TestRegexes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func timeMustParse(s string) *time.Time {
|
||||
t, _, _ := SearchDateFromLog(s)
|
||||
return &t
|
||||
}
|
||||
|
||||
func testRegexFromMap(t *testing.T, log string, regex *types.LogRegex) error {
|
||||
m := types.RegexMap{"test": regex}
|
||||
|
||||
|
||||
@@ -15,18 +15,26 @@ var SSTMap = types.RegexMap{
|
||||
// TODO: requested state from unknown node
|
||||
"RegexSSTRequestSuccess": &types.LogRegex{
|
||||
Regex: regexp.MustCompile("requested state transfer.*Selected"),
|
||||
InternalRegex: regexp.MustCompile("Member .* \\(" + regexNodeName + "\\) requested state transfer.*Selected .* \\(" + regexNodeName2 + "\\)\\("),
|
||||
InternalRegex: regexp.MustCompile("Member " + regexIdx + " \\(" + regexNodeName + "\\) requested state transfer.*Selected " + regexIdx + " \\(" + regexNodeName2 + "\\)\\("),
|
||||
Handler: func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) {
|
||||
|
||||
joiner := utils.ShortNodeName(submatches[groupNodeName])
|
||||
donor := utils.ShortNodeName(submatches[groupNodeName2])
|
||||
if utils.SliceContains(ctx.OwnNames, joiner) {
|
||||
ctx.SST.ResyncedFromNode = donor
|
||||
|
||||
sst := types.SST{
|
||||
Donor: donor,
|
||||
Joiner: joiner,
|
||||
}
|
||||
if utils.SliceContains(ctx.OwnNames, donor) {
|
||||
ctx.SST.ResyncingNode = joiner
|
||||
|
||||
// here this is a limitation of the handler signature
|
||||
// this is the only case when resorting to timestamps is necessary
|
||||
selectionTimestamp, _, ok := SearchDateFromLog(log)
|
||||
if ok {
|
||||
sst.SelectionTimestamp = &selectionTimestamp
|
||||
}
|
||||
|
||||
ctx.SSTs[donor] = sst
|
||||
|
||||
return ctx, func(ctx types.LogCtx) string {
|
||||
if utils.SliceContains(ctx.OwnNames, joiner) {
|
||||
return donor + utils.Paint(utils.GreenText, " will resync local node")
|
||||
@@ -58,18 +66,16 @@ var SSTMap = types.RegexMap{
|
||||
// 2022-12-24T03:28:22.444125Z 0 [Note] WSREP: 0.0 (name): State transfer to 2.0 (name2) complete.
|
||||
"RegexSSTComplete": &types.LogRegex{
|
||||
Regex: regexp.MustCompile("State transfer to.*complete"),
|
||||
InternalRegex: regexp.MustCompile("\\(" + regexNodeName + "\\): State transfer.*\\(" + regexNodeName2 + "\\) complete"),
|
||||
InternalRegex: regexp.MustCompile(regexIdx + " \\(" + regexNodeName + "\\): State transfer to " + regexIdx + " \\(" + regexNodeName2 + "\\) complete"),
|
||||
Handler: func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) {
|
||||
|
||||
donor := utils.ShortNodeName(submatches[groupNodeName])
|
||||
joiner := utils.ShortNodeName(submatches[groupNodeName2])
|
||||
displayType := "SST"
|
||||
if ctx.SST.Type != "" {
|
||||
displayType = ctx.SST.Type
|
||||
if ctx.SSTs[donor].Type != "" {
|
||||
displayType = ctx.SSTs[donor].Type
|
||||
}
|
||||
ctx.SST.Reset()
|
||||
|
||||
ctx = addOwnNameWithSSTMetadata(ctx, joiner, donor)
|
||||
delete(ctx.SSTs, donor)
|
||||
|
||||
return ctx, func(ctx types.LogCtx) string {
|
||||
if utils.SliceContains(ctx.OwnNames, joiner) {
|
||||
@@ -88,34 +94,34 @@ var SSTMap = types.RegexMap{
|
||||
// 2022-12-24T03:27:41.966118Z 0 [Note] WSREP: 0.0 (name): State transfer to -1.-1 (left the group) complete.
|
||||
"RegexSSTCompleteUnknown": &types.LogRegex{
|
||||
Regex: regexp.MustCompile("State transfer to.*left the group.*complete"),
|
||||
InternalRegex: regexp.MustCompile("\\(" + regexNodeName + "\\): State transfer.*\\(left the group\\) complete"),
|
||||
InternalRegex: regexp.MustCompile(regexIdx + " \\(" + regexNodeName + "\\): State transfer.*\\(left the group\\) complete"),
|
||||
Handler: func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) {
|
||||
|
||||
donor := utils.ShortNodeName(submatches[groupNodeName])
|
||||
ctx = addOwnNameWithSSTMetadata(ctx, "", donor)
|
||||
delete(ctx.SSTs, donor)
|
||||
return ctx, types.SimpleDisplayer(donor + utils.Paint(utils.RedText, " synced ??(node left)"))
|
||||
},
|
||||
},
|
||||
|
||||
"RegexSSTFailedUnknown": &types.LogRegex{
|
||||
Regex: regexp.MustCompile("State transfer to.*left the group.*failed"),
|
||||
InternalRegex: regexp.MustCompile("\\(" + regexNodeName + "\\): State transfer.*\\(left the group\\) failed"),
|
||||
InternalRegex: regexp.MustCompile(regexIdx + " \\(" + regexNodeName + "\\): State transfer.*\\(left the group\\) failed"),
|
||||
Handler: func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) {
|
||||
|
||||
donor := utils.ShortNodeName(submatches[groupNodeName])
|
||||
ctx = addOwnNameWithSSTMetadata(ctx, "", donor)
|
||||
delete(ctx.SSTs, donor)
|
||||
return ctx, types.SimpleDisplayer(donor + utils.Paint(utils.RedText, " failed to sync ??(node left)"))
|
||||
},
|
||||
},
|
||||
|
||||
"RegexSSTStateTransferFailed": &types.LogRegex{
|
||||
Regex: regexp.MustCompile("State transfer to.*failed:"),
|
||||
InternalRegex: regexp.MustCompile("\\(" + regexNodeName + "\\): State transfer.*\\(" + regexNodeName2 + "\\) failed"),
|
||||
InternalRegex: regexp.MustCompile(regexIdx + " \\(" + regexNodeName + "\\): State transfer to " + regexIdx + " \\(" + regexNodeName2 + "\\) failed"),
|
||||
Handler: func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) {
|
||||
|
||||
donor := utils.ShortNodeName(submatches[groupNodeName])
|
||||
joiner := utils.ShortNodeName(submatches[groupNodeName2])
|
||||
ctx = addOwnNameWithSSTMetadata(ctx, joiner, donor)
|
||||
delete(ctx.SSTs, donor)
|
||||
return ctx, types.SimpleDisplayer(donor + utils.Paint(utils.RedText, " failed to sync ") + joiner)
|
||||
},
|
||||
},
|
||||
@@ -128,6 +134,15 @@ var SSTMap = types.RegexMap{
|
||||
},
|
||||
},
|
||||
|
||||
"RegexSSTInitiating": &types.LogRegex{
|
||||
Regex: regexp.MustCompile("Initiating SST.IST transfer on DONOR side"),
|
||||
InternalRegex: regexp.MustCompile("DONOR side \\((?P<scriptname>[a-zA-Z0-9-_]*) --role 'donor' --address '" + regexNodeIP + ":(?P<sstport>[0-9]*)\\)"),
|
||||
Handler: func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) {
|
||||
|
||||
return ctx, types.SimpleDisplayer("init sst using " + submatches["scriptname"])
|
||||
},
|
||||
},
|
||||
|
||||
"RegexSSTCancellation": &types.LogRegex{
|
||||
Regex: regexp.MustCompile("Initiating SST cancellation"),
|
||||
Handler: func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) {
|
||||
@@ -140,7 +155,7 @@ var SSTMap = types.RegexMap{
|
||||
Regex: regexp.MustCompile("Proceeding with SST"),
|
||||
Handler: func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) {
|
||||
ctx.SetState("JOINER")
|
||||
ctx.SST.Type = "SST"
|
||||
ctx.SetSSTTypeMaybe("SST")
|
||||
|
||||
return ctx, types.SimpleDisplayer(utils.Paint(utils.YellowText, "receiving SST"))
|
||||
},
|
||||
@@ -152,13 +167,10 @@ var SSTMap = types.RegexMap{
|
||||
Handler: func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) {
|
||||
|
||||
ctx.SetState("DONOR")
|
||||
node := submatches[groupNodeIP]
|
||||
if ctx.SST.ResyncingNode == "" { // we should already have something at this point
|
||||
ctx.SST.ResyncingNode = node
|
||||
}
|
||||
joiner := submatches[groupNodeIP]
|
||||
|
||||
return ctx, func(ctx types.LogCtx) string {
|
||||
return utils.Paint(utils.YellowText, "SST to ") + types.DisplayNodeSimplestForm(ctx, node)
|
||||
return utils.Paint(utils.YellowText, "SST to ") + types.DisplayNodeSimplestForm(ctx, joiner)
|
||||
}
|
||||
},
|
||||
},
|
||||
@@ -181,8 +193,8 @@ var SSTMap = types.RegexMap{
|
||||
// TODO: sometimes, it's a hostname here
|
||||
InternalRegex: regexp.MustCompile("IST sender starting to serve " + regexNodeIPMethod + " sending [0-9]+-" + regexSeqno),
|
||||
Handler: func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) {
|
||||
ctx.SST.Type = "IST"
|
||||
ctx.SetState("DONOR")
|
||||
ctx.SetSSTTypeMaybe("IST")
|
||||
|
||||
seqno := submatches[groupSeqno]
|
||||
node := submatches[groupNodeIP]
|
||||
@@ -206,13 +218,13 @@ var SSTMap = types.RegexMap{
|
||||
startingseqno := submatches["startingseqno"]
|
||||
// if it's 0, it will go to SST without a doubt
|
||||
if startingseqno == "0" {
|
||||
ctx.SST.Type = "SST"
|
||||
ctx.SetSSTTypeMaybe("SST")
|
||||
msg += "SST"
|
||||
|
||||
// not totally correct, but need more logs to get proper pattern
|
||||
// in some cases it does IST before going with SST
|
||||
} else {
|
||||
ctx.SST.Type = "IST"
|
||||
ctx.SetSSTTypeMaybe("IST")
|
||||
msg += "IST"
|
||||
if seqno != "" {
|
||||
msg += "(seqno:" + seqno + ")"
|
||||
@@ -225,16 +237,25 @@ var SSTMap = types.RegexMap{
|
||||
"RegexFailedToPrepareIST": &types.LogRegex{
|
||||
Regex: regexp.MustCompile("Failed to prepare for incremental state transfer"),
|
||||
Handler: func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) {
|
||||
ctx.SST.Type = "SST"
|
||||
ctx.SetSSTTypeMaybe("SST")
|
||||
return ctx, types.SimpleDisplayer("IST is not applicable")
|
||||
},
|
||||
},
|
||||
|
||||
"RegexXtrabackupISTReceived": &types.LogRegex{
|
||||
Regex: regexp.MustCompile("xtrabackup_ist received from donor"),
|
||||
Handler: func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) {
|
||||
ctx.SetSSTTypeMaybe("IST")
|
||||
return ctx, types.SimpleDisplayer("IST running")
|
||||
},
|
||||
Verbosity: types.DebugMySQL, // that one is not really helpful, except for tooling constraints
|
||||
},
|
||||
|
||||
// could not find production examples yet, but it did exist in older version there also was "Bypassing state dump"
|
||||
"RegexBypassSST": &types.LogRegex{
|
||||
Regex: regexp.MustCompile("Bypassing SST"),
|
||||
Handler: func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) {
|
||||
ctx.SST.Type = "IST"
|
||||
ctx.SetSSTTypeMaybe("IST")
|
||||
return ctx, types.SimpleDisplayer("IST will be used")
|
||||
},
|
||||
},
|
||||
@@ -283,24 +304,7 @@ var SSTMap = types.RegexMap{
|
||||
},
|
||||
}
|
||||
|
||||
func addOwnNameWithSSTMetadata(ctx types.LogCtx, joiner, donor string) types.LogCtx {
|
||||
|
||||
var nameToAdd string
|
||||
|
||||
if ctx.State() == "JOINER" && joiner != "" {
|
||||
nameToAdd = joiner
|
||||
}
|
||||
if ctx.State() == "DONOR" && donor != "" {
|
||||
nameToAdd = donor
|
||||
}
|
||||
if nameToAdd != "" {
|
||||
ctx.AddOwnName(nameToAdd)
|
||||
}
|
||||
return ctx
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
2023-06-07T02:42:29.734960-06:00 0 [ERROR] WSREP: sst sent called when not SST donor, state SYNCED
|
||||
2023-06-07T02:42:00.234711-06:00 0 [Warning] WSREP: Protocol violation. JOIN message sender 0.0 (node1) is not in state transfer (SYNCED). Message ignored.
|
||||
|
||||
|
||||
@@ -14,7 +14,16 @@ func init() {
|
||||
var (
|
||||
shiftFunc = func(submatches map[string]string, ctx types.LogCtx, log string) (types.LogCtx, types.LogDisplayer) {
|
||||
|
||||
ctx.SetState(submatches["state2"])
|
||||
newState := submatches["state2"]
|
||||
ctx.SetState(newState)
|
||||
|
||||
if newState == "DONOR" || newState == "JOINER" {
|
||||
shiftTimestamp, _, ok := SearchDateFromLog(log)
|
||||
if ok {
|
||||
ctx.ConfirmSSTMetadata(shiftTimestamp)
|
||||
}
|
||||
}
|
||||
|
||||
log = utils.PaintForState(submatches["state1"], submatches["state1"]) + " -> " + utils.PaintForState(submatches["state2"], submatches["state2"])
|
||||
|
||||
return ctx, types.SimpleDisplayer(log)
|
||||
|
||||
Reference in New Issue
Block a user