diff --git a/bin/pt-online-schema-change b/bin/pt-online-schema-change index 31f09bd5..a1d968c1 100755 --- a/bin/pt-online-schema-change +++ b/bin/pt-online-schema-change @@ -5457,6 +5457,7 @@ use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); use constant PTDEBUG => $ENV{PTDEBUG} || 0; +use IndexLength; use Data::Dumper; $Data::Dumper::Indent = 1; @@ -5497,9 +5498,7 @@ sub new { my $nibble_sql = ($args{dml} ? "$args{dml} " : "SELECT ") . ($args{select} ? $args{select} - # : join(', ', map { $q->quote($_) } @cols)) - : join(', ', map{ $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? - "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_) } @cols)) + : join(', ', map { $q->quote($_) } @cols)) . " FROM $tbl->{name}" . ($where ? " WHERE $where" : '') . ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") @@ -5509,9 +5508,7 @@ sub new { my $explain_nibble_sql = "EXPLAIN SELECT " . ($args{select} ? $args{select} - # : join(', ', map { $q->quote($_) } @cols)) - : join(', ', map{ $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? - "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_) } @cols)) + : join(', ', map { $q->quote($_) } @cols)) . " FROM $tbl->{name}" . ($where ? " WHERE $where" : '') . ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") @@ -5540,43 +5537,40 @@ sub new { ); PTDEBUG && _d('Ascend params:', Dumper($asc)); - # my $force_concat_enums = $o->has('force-concat-enums') && $o->get('force-concat-enums'); - # my $i=0; - # for my $index (@{$index_cols}) { - # last if $args{n_chunk_index_cols} && $i >= $args{n_chunk_index_cols}; - # $i++; - # if ($tbl->{tbl_struct}->{type_for}->{$index} eq 'enum') { - # if ($tbl->{tbl_struct}->{defs}->{$index} =~ m/enum\s*\((.*?)\)/) { - # my @items = split(/,\s*/, $1); - # my $sorted = 1; # Asume the items list is sorted to later check if this is true - # for (my $i=1; $i < scalar(@items); $i++) { - # if ($items[$i-1] gt $items[$i]) { - # $sorted = 0; - # last; - # } - # } - # if (!$force_concat_enums && !$sorted) { - # die "The index " . $index . " in table " . $tbl->{name} . - # " has unsorted enum items.\nPlease read the documentation for the --force-concat-enums parameter\n"; - # } - # } - # } - # } + my $force_concat_enums = $o->has('force-concat-enums') && $o->get('force-concat-enums'); + my $i=0; + for my $index (@{$index_cols}) { + last if $args{n_chunk_index_cols} && $i >= $args{n_chunk_index_cols}; + $i++; + if ($tbl->{tbl_struct}->{type_for}->{$index} eq 'enum') { + if ($tbl->{tbl_struct}->{defs}->{$index} =~ m/enum\s*\((.*?)\)/) { + my @items = split(/,\s*/, $1); + my $sorted = 1; # Asume the items list is sorted to later check if this is true + for (my $i=1; $i < scalar(@items); $i++) { + if ($items[$i-1] gt $items[$i]) { + $sorted = 0; + last; + } + } + if (!$force_concat_enums && !$sorted) { + die "The index " . $index . " in table " . $tbl->{name} . + " has unsorted enum items.\nPlease read the documentation for the --force-concat-enums parameter\n"; + } + } + } + } my $from = "$tbl->{name} FORCE INDEX(`$index`)"; - #my $order_by = join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' && $force_concat_enums - # ? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols}); - my $order_by = join(', ', map {$q->quote($_)} @{$index_cols}); + my $order_by = join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' && $force_concat_enums + ? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols}); - #my $order_by_dec = join(' DESC,', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' && $force_concat_enums - # ? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols}); - my $order_by_dec = join(' DESC,', map {$q->quote($_)} @{$index_cols}); + my $order_by_dec = join(' DESC,', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' && $force_concat_enums + ? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols}); my $first_lb_sql = "SELECT /*!40001 SQL_NO_CACHE */ " - #. join(', ', map { $q->quote($_) } @{$asc->{scols}}) - . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}}) + . join(', ', map { $q->quote($_) } @{$asc->{scols}}) . " FROM $from" . ($where ? " WHERE $where" : '') . " ORDER BY $order_by" @@ -5588,8 +5582,7 @@ sub new { if ( $args{resume} ) { $resume_lb_sql = "SELECT /*!40001 SQL_NO_CACHE */ " - #. join(', ', map { $q->quote($_) } @{$asc->{scols}}) - . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}}) + . join(', ', map { $q->quote($_) } @{$asc->{scols}}) . " FROM $from" . " WHERE " . $asc->{boundaries}->{'>'} . ($where ? " AND ($where)" : '') @@ -5601,8 +5594,7 @@ sub new { my $last_ub_sql = "SELECT /*!40001 SQL_NO_CACHE */ " - #. join(', ', map { $q->quote($_) } @{$asc->{scols}}) - . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}}) + . join(', ', map { $q->quote($_) } @{$asc->{scols}}) . " FROM $from" . ($where ? " WHERE $where" : '') . " ORDER BY " @@ -5613,8 +5605,7 @@ sub new { my $ub_sql = "SELECT /*!40001 SQL_NO_CACHE */ " - #. join(', ', map { $q->quote($_) } @{$asc->{scols}}) - . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}}) + . join(', ', map { $q->quote($_) } @{$asc->{scols}}) . " FROM $from" . " WHERE " . $asc->{boundaries}->{'>='} . ($where ? " AND ($where)" : '') @@ -5626,8 +5617,7 @@ sub new { my $nibble_sql = ($args{dml} ? "$args{dml} " : "SELECT ") . ($args{select} ? $args{select} - # : join(', ', map { $q->quote($_) } @{$asc->{cols}})) - : join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{cols}})) + : join(', ', map { $q->quote($_) } @{$asc->{cols}})) . " FROM $from" . " WHERE " . $asc->{boundaries}->{'>='} # lower boundary . " AND " . $asc->{boundaries}->{'<='} # upper boundary @@ -5741,7 +5731,7 @@ sub next { if ( !$self->{have_rows} ) { $self->{nibbleno}++; PTDEBUG && _d('Nibble:', $self->{nibble_sth}->{Statement}, 'params:', - join(', ', (@{$self->{lower}} || [], @{$self->{upper} }||[]))); + join(', ', (@{$self->{lower} || []}, @{$self->{upper} || []}))); if ( my $callback = $self->{callbacks}->{exec_nibble} ) { $self->{have_rows} = $callback->(%callback_args); } @@ -5832,8 +5822,12 @@ sub set_boundary { } sub one_nibble { - my ($self) = @_; - return $self->{one_nibble}; + my ($self, $value) = @_; + if (@_ == 2) { + $self->{one_nibble} = $value; + } else { + return $self->{one_nibble}; + } } sub limit { @@ -5867,11 +5861,11 @@ sub row_estimate { sub can_nibble { my (%args) = @_; - my @required_args = qw(Cxn tbl chunk_size OptionParser TableParser); + my @required_args = qw(Cxn tbl chunk_size OptionParser TableParser Quoter); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } - my ($cxn, $tbl, $chunk_size, $o) = @args{@required_args}; + my ($cxn, $tbl, $chunk_size, $o, $q) = @args{@required_args}; my $where = $o->has('where') ? $o->get('where') : ''; @@ -5881,6 +5875,23 @@ sub can_nibble { where => $where, ); + my $can_get_keys; + if ($mysql_index) { + my $idx_len = IndexLength->new(Quoter => $q); + my ($key_len, $key) = $idx_len->index_length( + Cxn => $args{Cxn}, + tbl => $tbl, + index => $mysql_index, + n_index_cols => $o->get('chunk-index-columns'), + ); + + if ( !$key || !$key_len || lc($key) ne lc($mysql_index)) { + $can_get_keys = 0; + } else { + $can_get_keys = 1; + } + } + if ( !$where ) { $mysql_index = undef; } @@ -5889,6 +5900,10 @@ sub can_nibble { my $one_nibble = !defined $args{one_nibble} || $args{one_nibble} ? $row_est <= $chunk_size * $chunk_size_limit : 0; + if (!$can_get_keys) { + $one_nibble = 1; + } + PTDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no'); if ( $args{resume} @@ -6140,7 +6155,7 @@ sub _next_boundaries { PTDEBUG && _d($self->{ub_sth}->{Statement}, 'params:', - join(', ', @{$self->{lower}} || []), $self->{limit}); + join(', ', @{$self->{lower}}), $self->{limit}); $self->{ub_sth}->execute(@{$self->{lower}}, $self->{limit}); my $boundary = $self->{ub_sth}->fetchall_arrayref(); PTDEBUG && _d('Next boundary:', Dumper($boundary)); @@ -6181,9 +6196,6 @@ sub identical_boundaries { if scalar @$b1 != scalar @$b2; # shouldn't happen my $n_vals = scalar @$b1; for my $i ( 0..($n_vals-1) ) { - next if (!defined($b1->[$i]) && !defined($b2->[$i])); - return 0 if (!defined($b1->[$i]) && defined($b2->[$i])); # diff - return 0 if (defined($b1->[$i]) && !defined($b2->[$i])); # diff return 0 if $b1->[$i] ne $b2->[$i]; # diff } return 1; @@ -9579,6 +9591,7 @@ sub main { # a single chunk on all slaves. E.g. if a slave is out of sync # and has a lot more rows than the master, single chunking on the # master could cause the slave to choke. + my $failed_to_nibble; if ( $nibble_iter->one_nibble() ) { PTDEBUG && _d('Getting table row estimate on replicas'); my @too_large; @@ -9605,10 +9618,13 @@ sub main { . ($tbl->{chunk_size} * $limit) . " rows (chunk size=$tbl->{chunk_size}" . " * chunk size limit=$limit).\n"; - die ts($msg); + warn $msg; + $failed_to_nibble = 1; + $nibble_iter->one_nibble(0); # disable nibbling for this table + # die ts($msg); } } - else { # chunking the table + if (!$nibble_iter->one_nibble() || $failed_to_nibble) { # chunking the table if ( $o->get('check-plan') ) { my $idx_len = new IndexLength(Quoter => $q); my ($key_len, $key) = $idx_len->index_length( diff --git a/bin/pt-table-checksum b/bin/pt-table-checksum index 8ccabf4d..cc15207b 100755 --- a/bin/pt-table-checksum +++ b/bin/pt-table-checksum @@ -6593,6 +6593,7 @@ sub next { } $self->{rowno} = 0; $self->{have_rows} = 0; + } PTDEBUG && _d('Done nibbling'); @@ -6657,8 +6658,12 @@ sub set_boundary { } sub one_nibble { - my ($self) = @_; - return $self->{one_nibble}; + my ($self, $value) = @_; + if (@_ == 2) { + $self->{one_nibble} = $value; + } else { + return $self->{one_nibble}; + } } sub limit { @@ -6706,15 +6711,7 @@ sub can_nibble { where => $where, ); - if ( !$where ) { - $mysql_index = undef; - } - - my $chunk_size_limit = $o->get('chunk-size-limit') || 1; - my $one_nibble = !defined $args{one_nibble} || $args{one_nibble} - ? $row_est <= $chunk_size * $chunk_size_limit - : 0; - + my $can_get_keys; if ($mysql_index) { my $idx_len = IndexLength->new(Quoter => $q); my ($key_len, $key) = $idx_len->index_length( @@ -6725,10 +6722,24 @@ sub can_nibble { ); if ( !$key || !$key_len || lc($key) ne lc($mysql_index)) { - $one_nibble = 1; + $can_get_keys = 0; + } else { + $can_get_keys = 1; } } + if ( !$where ) { + $mysql_index = undef; + } + + my $chunk_size_limit = $o->get('chunk-size-limit') || 1; + my $one_nibble = !defined $args{one_nibble} || $args{one_nibble} + ? $row_est <= $chunk_size * $chunk_size_limit + : 0; + if (!$can_get_keys) { + $one_nibble = 1; + } + PTDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no'); if ( $args{resume} @@ -6906,7 +6917,7 @@ sub _get_bounds { if ( defined $nibble->{lower_boundary} && defined $nibble->{upper_boundary} ) { my $sth = $dbh->prepare($self->{resume_lb_sql}); - my @ub = $self->{Quoter}->deserialize_list($nibble->{upper_boundary}); + my @ub = split ',', $nibble->{upper_boundary}; PTDEBUG && _d($sth->{Statement}, 'params:', @ub); $sth->execute(@ub); $self->{next_lower} = $sth->fetchrow_arrayref(); diff --git a/lib/IndexLength.pm b/lib/IndexLength.pm index c2f12055..d160a257 100644 --- a/lib/IndexLength.pm +++ b/lib/IndexLength.pm @@ -99,7 +99,7 @@ sub _get_first_values { } my ($cxn, $tbl, $index, $n_index_cols) = @args{@required_args}; - my $q = $self->{quoter}; + my $q = $self->{Quoter}; # Select just the index columns. my $index_struct = $tbl->{tbl_struct}->{keys}->{$index}; diff --git a/lib/NibbleIterator.pm b/lib/NibbleIterator.pm index 50784af9..7fa31eed 100644 --- a/lib/NibbleIterator.pm +++ b/lib/NibbleIterator.pm @@ -121,6 +121,7 @@ sub new { }; } else { + my $parms = _nibble_params($nibble_params, $tbl, %args, \@cols, $chunk_size, $where, \%comments, $q); my $index = $nibble_params->{index}; # brevity my $index_cols = $tbl->{tbl_struct}->{keys}->{$index}->{cols}; @@ -293,9 +294,198 @@ sub new { $self->{pause_file} = $nibble_params->{pause_file}; $self->{sleep} = $args{sleep} || 60; + $self->{chunk_size} = $chunk_size; + $self->{comments} = %comments; + $self->{cols} = \@cols; + $self->{where} = $where; + $self->{tbl} = $tbl; + return bless $self, $class; } +sub _one_nibble { + my ($self, %args) = @_; + my $q = new Quoter(); + my $cols = $self->{cols}; + my $where = $self->{where}; + my $tbl = $self->{tbl}; + my %comments = $self->{comments}; + + # If the chunk size is >= number of rows in table, then we don't + # need to chunk; we can just select all rows, in order, at once. + my $nibble_sql + = ($args{dml} ? "$args{dml} " : "SELECT ") + . ($args{select} ? $args{select} + : join(', ', map { $q->quote($_) } @$cols)) + . " FROM $tbl->{name}" + . ($where ? " WHERE $where" : '') + . ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") + . " /*$comments{bite}*/"; + PTDEBUG && _d('One nibble statement:', $nibble_sql); + + my $explain_nibble_sql + = "EXPLAIN SELECT " + . ($args{select} ? $args{select} + : join(', ', map { $q->quote($_) } @$cols)) + . " FROM $tbl->{name}" + . ($where ? " WHERE $where" : '') + . ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") + . " /*explain $comments{bite}*/"; + PTDEBUG && _d('Explain one nibble statement:', $explain_nibble_sql); + + return { + one_nibble => 1, + limit => 0, + nibble_sql => $nibble_sql, + explain_nibble_sql => $explain_nibble_sql, + }; +} + +sub _nibble_params { + my ($nibble_params, $tbl, %args, $cols, $chunk_size, $where, %comments, $q) = @_; + my $index = $nibble_params->{index}; # brevity + my $index_cols = $tbl->{tbl_struct}->{keys}->{$index}->{cols}; + + # Figure out how to nibble the table with the index. + my $asc = $args{TableNibbler}->generate_asc_stmt( + %args, + tbl_struct => $tbl->{tbl_struct}, + index => $index, + n_index_cols => $args{n_chunk_index_cols}, + cols => $cols, + asc_only => 1, + ); + PTDEBUG && _d('Ascend params:', Dumper($asc)); + + # Check if enum fields items are sorted or not. + # If they are sorted we can skip adding CONCAT to improve the queries eficiency. + my $force_concat_enums; + + # Make SQL statements, prepared on first call to next(). FROM and + # ORDER BY are the same for all statements. FORCE IDNEX and ORDER BY + # are needed to ensure deterministic nibbling. + + my $from = "$tbl->{name} FORCE INDEX(`$index`)"; + my $order_by = join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' && $force_concat_enums + ? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols}); + + my $order_by_dec = join(' DESC,', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' && $force_concat_enums + ? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols}); + + # The real first row in the table. Usually we start nibbling from + # this row. Called once in _get_bounds(). + my $first_lb_sql + = "SELECT /*!40001 SQL_NO_CACHE */ " + . join(', ', map { $q->quote($_) } @{$asc->{scols}}) + . " FROM $from" + . ($where ? " WHERE $where" : '') + . " ORDER BY $order_by" + . " LIMIT 1" + . " /*first lower boundary*/"; + PTDEBUG && _d('First lower boundary statement:', $first_lb_sql); + + # If we're resuming, this fetches the effective first row, which + # should differ from the real first row. Called once in _get_bounds(). + my $resume_lb_sql; + if ( $args{resume} ) { + $resume_lb_sql + = "SELECT /*!40001 SQL_NO_CACHE */ " + . join(', ', map { $q->quote($_) } @{$asc->{scols}}) + . " FROM $from" + . " WHERE " . $asc->{boundaries}->{'>'} + . ($where ? " AND ($where)" : '') + . " ORDER BY $order_by" + . " LIMIT 1" + . " /*resume lower boundary*/"; + PTDEBUG && _d('Resume lower boundary statement:', $resume_lb_sql); + } + + # The nibbles are inclusive, so we need to fetch the real last row + # in the table. Saved as boundary last_upper and used as boundary + # upper in some cases. Called once in _get_bounds(). + my $last_ub_sql + = "SELECT /*!40001 SQL_NO_CACHE */ " + . join(', ', map { $q->quote($_) } @{$asc->{scols}}) + . " FROM $from" + . ($where ? " WHERE $where" : '') + . " ORDER BY " + . $order_by_dec . ' DESC' + . " LIMIT 1" + . " /*last upper boundary*/"; + PTDEBUG && _d('Last upper boundary statement:', $last_ub_sql); + + # Nibbles are inclusive, so for a..z, the nibbles are: a-e, f-j, k-o, p-t, + # u-y, and z. This complicates getting the next upper boundary because + # if we use either (col >= lb AND col < ub) or (col > lb AND col <= ub) + # in nibble_sql (below), then that fails for either the last or first + # nibble respectively. E.g. (col >= z AND col < z) doesn't work, nor + # does (col > a AND col <= e). Hence the fancy LIMIT 2 which returns + # the upper boundary for the current nibble *and* the lower boundary + # for the next nibble. See _next_boundaries(). + my $ub_sql + = "SELECT /*!40001 SQL_NO_CACHE */ " + . join(', ', map { $q->quote($_) } @{$asc->{scols}}) + . " FROM $from" + . " WHERE " . $asc->{boundaries}->{'>='} + . ($where ? " AND ($where)" : '') + . " ORDER BY $order_by" + . " LIMIT ?, 2" + . " /*next chunk boundary*/"; + PTDEBUG && _d('Upper boundary statement:', $ub_sql); + + # This statement does the actual nibbling work; its rows are returned + # to the caller via next(). + my $nibble_sql + = ($args{dml} ? "$args{dml} " : "SELECT ") + . ($args{select} ? $args{select} + : join(', ', map { $q->quote($_) } @{$asc->{cols}})) + . " FROM $from" + . " WHERE " . $asc->{boundaries}->{'>='} # lower boundary + . " AND " . $asc->{boundaries}->{'<='} # upper boundary + . ($where ? " AND ($where)" : '') + . ($args{order_by} ? " ORDER BY $order_by" : "") + . ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") + . " /*$comments{nibble}*/"; + PTDEBUG && _d('Nibble statement:', $nibble_sql); + + my $explain_nibble_sql + = "EXPLAIN SELECT " + . ($args{select} ? $args{select} + : join(', ', map { $q->quote($_) } @{$asc->{cols}})) + . " FROM $from" + . " WHERE " . $asc->{boundaries}->{'>='} # lower boundary + . " AND " . $asc->{boundaries}->{'<='} # upper boundary + . ($where ? " AND ($where)" : '') + . ($args{order_by} ? " ORDER BY $order_by" : "") + . ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") + . " /*explain $comments{nibble}*/"; + PTDEBUG && _d('Explain nibble statement:', $explain_nibble_sql); + + my $limit = $chunk_size - 1; + PTDEBUG && _d('Initial chunk size (LIMIT):', $limit); + + my $params = { + index => $index, + limit => $limit, + first_lb_sql => $first_lb_sql, + last_ub_sql => $last_ub_sql, + ub_sql => $ub_sql, + nibble_sql => $nibble_sql, + explain_first_lb_sql => "EXPLAIN $first_lb_sql", + explain_ub_sql => "EXPLAIN $ub_sql", + explain_nibble_sql => $explain_nibble_sql, + resume_lb_sql => $resume_lb_sql, + sql => { + columns => $asc->{scols}, + from => $from, + where => $where, + boundaries => $asc->{boundaries}, + order_by => $order_by, + }, + }; + return $params; +} + sub next { my ($self) = @_; @@ -453,8 +643,12 @@ sub set_boundary { } sub one_nibble { - my ($self) = @_; - return $self->{one_nibble}; + my ($self, $value) = @_; + if (@_ == 2) { + $self->{one_nibble} = $value; + } else { + return $self->{one_nibble}; + } } sub limit { diff --git a/t/lib/NibbleIterator.t b/t/lib/NibbleIterator.t index 10f5548e..2431cadf 100644 --- a/t/lib/NibbleIterator.t +++ b/t/lib/NibbleIterator.t @@ -36,7 +36,6 @@ my $dp = new DSNParser(opts=>$dsn_opts); my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp); my $dbh = $sb->get_dbh_for('master'); -plan skip_all => 'Cannot connect to sandbox master'; if ( !$dbh ) { plan skip_all => 'Cannot connect to sandbox master'; } else {