percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-12-11 02:04:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '3.x' into PT-2309_pt-table-sync_hits_an_error_when_the_primary_key_is_uuid_binary

Browse Source
This commit is contained in:
Sveta Smirnova
2024-04-18 13:16:19 +03:00
parent eac0b21407 18d4fe9183
commit 6632ccc044
27 changed files with 190 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.typos.toml
View File

@@ -61,6 +61,7 @@ extend-ignore-re = [
"RegexXtrabackupISTReceived" = "RegexXtrabackupISTReceived"
"START_ND_SUMMARY" = "START_ND_SUMMARY"
"START_ND_TOOLTIPS" = "START_ND_TOOLTIPS"
"thr" = "thr" # common abbreviation
"TOI" = "TOI"
"UNIONed" = "UNIONed"
"UNIONs" = "UNIONs"

3
bin/pt-archiver
View File

@@ -3781,7 +3781,8 @@ sub recurse_to_slaves {
my $slave_dsn = $dsn;
if ($slave_user) {
$slave_dsn->{u} = $slave_user;
PTDEBUG && _d("Using slave user $slave_user on ".$slave_dsn->{h}.":".$slave_dsn->{P});
PTDEBUG && _d("Using slave user $slave_user on "
. $slave_dsn->{h} . ":" . ( $slave_dsn->{P} ? $slave_dsn->{P} : ""));
}
if ($slave_password) {
$slave_dsn->{p} = $slave_password;

3
bin/pt-heartbeat
View File

@@ -242,7 +242,8 @@ sub recurse_to_slaves {
my $slave_dsn = $dsn;
if ($slave_user) {
$slave_dsn->{u} = $slave_user;
PTDEBUG && _d("Using slave user $slave_user on ".$slave_dsn->{h}.":".$slave_dsn->{P});
PTDEBUG && _d("Using slave user $slave_user on "
. $slave_dsn->{h} . ":" . ( $slave_dsn->{P} ? $slave_dsn->{P} : ""));
}
if ($slave_password) {
$slave_dsn->{p} = $slave_password;

4
bin/pt-ioprofile
View File

@@ -408,7 +408,7 @@ _parse_command_line() {
fi
if $(echo $opt | grep '^[a-z-][a-z-]*=' >/dev/null 2>&1); then
val="$(echo $opt | awk -F= '{print $2}')"
val="$(echo "$opt" | awk '{ st = index($0,"="); print substr($0, st+1)}')"
opt="$(echo $opt | awk -F= '{print $1}')"
fi
@@ -453,7 +453,7 @@ _parse_command_line() {
val=$(size_to_bytes $val)
fi
eval "OPT_$opt"="'$val'"
eval "OPT_$opt"='$val'
opt=""
val=""

3
bin/pt-kill
View File

@@ -4058,7 +4058,8 @@ sub recurse_to_slaves {
my $slave_dsn = $dsn;
if ($slave_user) {
$slave_dsn->{u} = $slave_user;
PTDEBUG && _d("Using slave user $slave_user on ".$slave_dsn->{h}.":".$slave_dsn->{P});
PTDEBUG && _d("Using slave user $slave_user on "
. $slave_dsn->{h} . ":" . ( $slave_dsn->{P} ? $slave_dsn->{P} : ""));
}
if ($slave_password) {
$slave_dsn->{p} = $slave_password;

4
bin/pt-mext
View File

@@ -449,7 +449,7 @@ _parse_command_line() {
fi
if $(echo $opt | grep '^[a-z-][a-z-]*=' >/dev/null 2>&1); then
val="$(echo $opt | awk -F= '{print $2}')"
val="$(echo "$opt" | awk '{ st = index($0,"="); print substr($0, st+1)}')"
opt="$(echo $opt | awk -F= '{print $1}')"
fi
@@ -494,7 +494,7 @@ _parse_command_line() {
val=$(size_to_bytes $val)
fi
eval "OPT_$opt"="'$val'"
eval "OPT_$opt"='$val'
opt=""
val=""

4
bin/pt-mysql-summary
View File

@@ -410,7 +410,7 @@ _parse_command_line() {
fi
if $(echo $opt | grep '^[a-z-][a-z-]*=' >/dev/null 2>&1); then
val="$(echo $opt | awk -F= '{print $2}')"
val="$(echo "$opt" | awk '{ st = index($0,"="); print substr($0, st+1)}')"
opt="$(echo $opt | awk -F= '{print $1}')"
fi
@@ -455,7 +455,7 @@ _parse_command_line() {
val=$(size_to_bytes $val)
fi
eval "OPT_$opt"="'$val'"
eval "OPT_$opt"='$val'
opt=""
val=""

3
bin/pt-online-schema-change
View File

@@ -4346,7 +4346,8 @@ sub recurse_to_slaves {
my $slave_dsn = $dsn;
if ($slave_user) {
$slave_dsn->{u} = $slave_user;
PTDEBUG && _d("Using slave user $slave_user on ".$slave_dsn->{h}.":".$slave_dsn->{P});
PTDEBUG && _d("Using slave user $slave_user on "
. $slave_dsn->{h} . ":" . ( $slave_dsn->{P} ? $slave_dsn->{P} : ""));
}
if ($slave_password) {
$slave_dsn->{p} = $slave_password;

4
bin/pt-pmp
View File

@@ -451,7 +451,7 @@ _parse_command_line() {
fi
if $(echo $opt | grep '^[a-z-][a-z-]*=' >/dev/null 2>&1); then
val="$(echo $opt | awk -F= '{print $2}')"
val="$(echo "$opt" | awk '{ st = index($0,"="); print substr($0, st+1)}')"
opt="$(echo $opt | awk -F= '{print $1}')"
fi
@@ -496,7 +496,7 @@ _parse_command_line() {
val=$(size_to_bytes $val)
fi
eval "OPT_$opt"="'$val'"
eval "OPT_$opt"='$val'
opt=""
val=""

3
bin/pt-query-digest
View File

@@ -10652,7 +10652,8 @@ sub recurse_to_slaves {
my $slave_dsn = $dsn;
if ($slave_user) {
$slave_dsn->{u} = $slave_user;
PTDEBUG && _d("Using slave user $slave_user on ".$slave_dsn->{h}.":".$slave_dsn->{P});
PTDEBUG && _d("Using slave user $slave_user on "
. $slave_dsn->{h} . ":" . ( $slave_dsn->{P} ? $slave_dsn->{P} : ""));
}
if ($slave_password) {
$slave_dsn->{p} = $slave_password;

4
bin/pt-sift
View File

@@ -449,7 +449,7 @@ _parse_command_line() {
fi
if $(echo $opt | grep '^[a-z-][a-z-]*=' >/dev/null 2>&1); then
val="$(echo $opt | awk -F= '{print $2}')"
val="$(echo "$opt" | awk '{ st = index($0,"="); print substr($0, st+1)}')"
opt="$(echo $opt | awk -F= '{print $1}')"
fi
@@ -494,7 +494,7 @@ _parse_command_line() {
val=$(size_to_bytes $val)
fi
eval "OPT_$opt"="'$val'"
eval "OPT_$opt"='$val'
opt=""
val=""

3
bin/pt-slave-find
View File

@@ -2383,7 +2383,8 @@ sub recurse_to_slaves {
my $slave_dsn = $dsn;
if ($slave_user) {
$slave_dsn->{u} = $slave_user;
PTDEBUG && _d("Using slave user $slave_user on ".$slave_dsn->{h}.":".$slave_dsn->{P});
PTDEBUG && _d("Using slave user $slave_user on "
. $slave_dsn->{h} . ":" . ( $slave_dsn->{P} ? $slave_dsn->{P} : ""));
}
if ($slave_password) {
$slave_dsn->{p} = $slave_password;

3
bin/pt-slave-restart
View File

@@ -2794,7 +2794,8 @@ sub recurse_to_slaves {
my $slave_dsn = $dsn;
if ($slave_user) {
$slave_dsn->{u} = $slave_user;
PTDEBUG && _d("Using slave user $slave_user on ".$slave_dsn->{h}.":".$slave_dsn->{P});
PTDEBUG && _d("Using slave user $slave_user on "
. $slave_dsn->{h} . ":" . ( $slave_dsn->{P} ? $slave_dsn->{P} : ""));
}
if ($slave_password) {
$slave_dsn->{p} = $slave_password;

4
bin/pt-stalk
View File

@@ -462,7 +462,7 @@ _parse_command_line() {
fi
if $(echo $opt | grep '^[a-z-][a-z-]*=' >/dev/null 2>&1); then
val="$(echo $opt | awk -F= '{print $2}')"
val="$(echo "$opt" | awk '{ st = index($0,"="); print substr($0, st+1)}')"
opt="$(echo $opt | awk -F= '{print $1}')"
fi
@@ -507,7 +507,7 @@ _parse_command_line() {
val=$(size_to_bytes $val)
fi
eval "OPT_$opt"="'$val'"
eval "OPT_$opt"='$val'
opt=""
val=""

4
bin/pt-summary
View File

@@ -417,7 +417,7 @@ _parse_command_line() {
fi
if $(echo $opt | grep '^[a-z-][a-z-]*=' >/dev/null 2>&1); then
val="$(echo $opt | awk -F= '{print $2}')"
val="$(echo "$opt" | awk '{ st = index($0,"="); print substr($0, st+1)}')"
opt="$(echo $opt | awk -F= '{print $1}')"
fi
@@ -462,7 +462,7 @@ _parse_command_line() {
val=$(size_to_bytes $val)
fi
eval "OPT_$opt"="'$val'"
eval "OPT_$opt"='$val'
opt=""
val=""

3
bin/pt-table-checksum
View File

@@ -5298,7 +5298,8 @@ sub recurse_to_slaves {
my $slave_dsn = $dsn;
if ($slave_user) {
$slave_dsn->{u} = $slave_user;
PTDEBUG && _d("Using slave user $slave_user on ".$slave_dsn->{h}.":".$slave_dsn->{P});
PTDEBUG && _d("Using slave user $slave_user on "
. $slave_dsn->{h} . ":" . ( $slave_dsn->{P} ? $slave_dsn->{P} : ""));
}
if ($slave_password) {
$slave_dsn->{p} = $slave_password;

3
bin/pt-table-sync
View File

@@ -6812,7 +6812,8 @@ sub recurse_to_slaves {
my $slave_dsn = $dsn;
if ($slave_user) {
$slave_dsn->{u} = $slave_user;
PTDEBUG && _d("Using slave user $slave_user on ".$slave_dsn->{h}.":".$slave_dsn->{P});
PTDEBUG && _d("Using slave user $slave_user on "
. $slave_dsn->{h} . ":" . ( $slave_dsn->{P} ? $slave_dsn->{P} : ""));
}
if ($slave_password) {
$slave_dsn->{p} = $slave_password;

2
go.mod
View File

@@ -26,7 +26,7 @@ require (
github.com/sirupsen/logrus v1.9.3
github.com/stretchr/testify v1.9.0
github.com/xlab/treeprint v1.2.0
go.mongodb.org/mongo-driver v1.14.0
go.mongodb.org/mongo-driver v1.15.0
golang.org/x/crypto v0.22.0
golang.org/x/exp v0.0.0-20230321023759-10a507213a29
gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22

4
go.sum
View File

@@ -118,8 +118,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80=
go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=
go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc=
go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=

3
lib/MasterSlave.pm
View File

@@ -181,7 +181,8 @@ sub recurse_to_slaves {
my $slave_dsn = $dsn;
if ($slave_user) {
$slave_dsn->{u} = $slave_user;
PTDEBUG && _d("Using slave user $slave_user on ".$slave_dsn->{h}.":".$slave_dsn->{P});
PTDEBUG && _d("Using slave user $slave_user on "
. $slave_dsn->{h} . ":" . ( $slave_dsn->{P} ? $slave_dsn->{P} : ""));
}
if ($slave_password) {
$slave_dsn->{p} = $slave_password;

4
lib/bash/parse_options.sh
View File

@@ -475,7 +475,7 @@ _parse_command_line() {
# Split opt=val pair.
if $(echo $opt | grep '^[a-z-][a-z-]*=' >/dev/null 2>&1); then
val="$(echo $opt | awk -F= '{print $2}')"
val="$(echo "$opt" | awk '{ st = index($0,"="); print substr($0, st+1)}')"
opt="$(echo $opt | awk -F= '{print $1}')"
fi
@@ -533,7 +533,7 @@ _parse_command_line() {
fi
# Re-eval the option to update its global variable value.
eval "OPT_$opt"="'$val'"
eval "OPT_$opt"='$val'
opt=""
val=""

2
src/go/docker-compose.yml
View File

@@ -186,7 +186,7 @@ services:
image: ${TEST_MONGODB_IMAGE:-mongo:4.2}
ports:
- "${TEST_MONGODB_STANDALONE_PORT:-27017}:27017"
command: mongod --replSet rs1 --shardsvr --port 27017 --oplogSize 16
command: mongod --port 27017 --oplogSize 16
postgres9:
image: ${POSTGRE_IMAGE:-postgres:9.6}
container_name: go_postgres9_1

10
src/go/lib/versioncheck/version_check.go
View File

@@ -73,15 +73,15 @@ func checkUpdates(url string, timeout time.Duration, toolName, version string) (
if err != nil {
return "", err
}
advices := []Advice{}
err = json.Unmarshal(body, &advices)
var advice []Advice
err = json.Unmarshal(body, &advice)
if err != nil {
return "", err
}
for _, advice := range advices {
if advice.ToolName == PERCONA_TOOLKIT {
return advice.Advice, nil
for _, a := range advice {
if a.ToolName == PERCONA_TOOLKIT {
return a.Advice, nil
}
}

4
src/go/lib/versioncheck/version_check_test.go
View File

@@ -16,7 +16,7 @@ func TestCheckUpdates(t *testing.T) {
body, _ := ioutil.ReadAll(r.Body)
m := strings.Split(string(body), ";")
advices := []Advice{
advice := []Advice{
{
Hash: m[0],
ToolName: m[1],
@@ -24,7 +24,7 @@ func TestCheckUpdates(t *testing.T) {
},
}
buf, _ := json.Marshal(advices)
buf, _ := json.Marshal(advice)
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, string(buf))
}))

6
src/go/pt-mongodb-summary/main.go
View File

@@ -374,14 +374,14 @@ func getHostInfo(ctx context.Context, client *mongo.Client) (*hostInfo, error) {
}
cmdOpts := proto.CommandLineOptions{}
query := primitive.D{{Key: "getCmdLineOpts", Value: 1}, {Key: "recordStats", Value: 1}}
query := primitive.D{{Key: "getCmdLineOpts", Value: 1}}
err := client.Database("admin").RunCommand(ctx, query).Decode(&cmdOpts)
if err != nil {
return nil, errors.Wrap(err, "cannot get command line options")
}
ss := proto.ServerStatus{}
query = primitive.D{{Key: "serverStatus", Value: 1}, {Key: "recordStats", Value: 1}}
query = primitive.D{{Key: "serverStatus", Value: 1}}
if err := client.Database("admin").RunCommand(ctx, query).Decode(&ss); err != nil {
return nil, errors.Wrap(err, "GetHostInfo.serverStatus")
}
@@ -528,7 +528,6 @@ func getSecuritySettings(ctx context.Context, client *mongo.Client, ver string)
cmdOpts := proto.CommandLineOptions{}
err = client.Database("admin").RunCommand(ctx, primitive.D{
{Key: "getCmdLineOpts", Value: 1},
{Key: "recordStats", Value: 1},
}).Decode(&cmdOpts)
if err != nil {
return nil, errors.Wrap(err, "cannot get command line options")
@@ -633,7 +632,6 @@ func getOpCountersStats(ctx context.Context, client *mongo.Client, count int,
err := client.Database("admin").RunCommand(ctx, primitive.D{
{Key: "serverStatus", Value: 1},
{Key: "recordStats", Value: 1},
}).Decode(&ss)
if err != nil {
return nil, err

77
t/pt-mysql-summary/pt-2327.t Normal file
View File

@@ -0,0 +1,77 @@
#!/usr/bin/env perl
BEGIN {
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
};
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use PerconaTest;
use Sandbox;
use DSNParser;
require VersionParser;
use Test::More;
local $ENV{PTDEBUG} = "";
my $dp = new DSNParser(opts=>$dsn_opts);
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
my $dbh = $sb->get_dbh_for('master');
my $cnf = '/tmp/12345/my.sandbox.cnf';
my $output;
if ( !$dbh ) {
plan skip_all => 'Cannot connect to sandbox master';
}
else {
plan tests => 5;
}
$sb->do_as_root("master", q/create user pt2302 identified by "root_'f<=*password"/);
$sb->do_as_root("master", q/grant all on *.* to pt2302/);
my $cmd = "$trunk/bin/pt-mysql-summary --sleep 1 -- --defaults-file=$cnf --user=pt2302 --password=\"root_'f<=*password\"";
$output = `$cmd 2>&1`;
unlike(
$output,
qr/eval: Syntax error: Unterminated quoted string/s,
"pt-mysql-summary does not stop with password containing an apostrophe"
);
unlike(
$output,
qr/Access denied for user/s,
"pt-mysql-summary works fine with password containing an apostrophe"
);
$sb->do_as_root("master", q/drop user pt2302/);
$sb->do_as_root("master", q/create user pt2302 identified by 'root_"f<=*password'/);
$sb->do_as_root("master", q/grant all on *.* to pt2302/);
$cmd = "$trunk/bin/pt-mysql-summary --sleep 1 -- --defaults-file=$cnf --user=pt2302 --password='root_\"f<=*password'";
$output = `$cmd 2>&1`;
unlike(
$output,
qr/eval: Syntax error: Unterminated quoted string/s,
"pt-mysql-summary does not stop with password containing a quote"
);
unlike(
$output,
qr/Access denied for user/s,
"pt-mysql-summary works fine with password containing a quote"
);
# #############################################################################
# Done.
# #############################################################################
$sb->do_as_root("master", q/drop user pt2302/);
$sb->wipe_clean($dbh);
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
exit;

63
t/pt-online-schema-change/pt-2231.t Normal file
View File

@@ -0,0 +1,63 @@
#!/usr/bin/env perl
BEGIN {
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
};
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Test::More;
use PerconaTest;
use Sandbox;
require "$trunk/bin/pt-online-schema-change";
my $dp = new DSNParser(opts=>$dsn_opts);
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
my $master_dbh = $sb->get_dbh_for('master');
my $slave1_dbh = $sb->get_dbh_for('slave1');
my $slave2_dbh = $sb->get_dbh_for('slave2');
if ( !$master_dbh ) {
plan skip_all => 'Cannot connect to sandbox master';
}
elsif ( !$slave1_dbh ) {
plan skip_all => 'Cannot connect to sandbox slave1';
}
elsif ( !$slave1_dbh ) {
plan skip_all => 'Cannot connect to sandbox slave2';
}
else {
plan tests => 2;
}
$sb->load_file('master', "t/pt-online-schema-change/samples/basic_no_fks.sql");
$sb->wait_for_slaves();
# Save original PTDEBUG env because we modify it below.
my $dbg = $ENV{PTDEBUG};
$ENV{PTDEBUG} = 1;
my $output = `$trunk/bin/pt-online-schema-change h=localhost,S=/tmp/12345/mysql_sandbox12345.sock,D=pt_osc,t=t --user=msandbox --password=msandbox --slave-user=msandbox --slave-password=msandbox --alter "FORCE" --recursion-method=processlist --no-check-replication-filters --no-check-alter --no-check-plan --chunk-index=PRIMARY --no-version-check --execute 2>&1`;
unlike(
$output,
qr/Use of uninitialized value in concatenation (.) or string/,
'No error with PTDEBUG output'
) or diag($output);
# Restore PTDEBUG env.
delete $ENV{PTDEBUG};
$ENV{PTDEBUG} = $dbg || 0;
# #############################################################################
# Done.
# #############################################################################
$sb->wipe_clean($master_dbh);
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
exit;