mirror of
https://github.com/percona/percona-toolkit.git
synced 2025-10-21 10:17:18 +00:00
PT-221 pt-table-sync support for MyRocks
This commit is contained in:
Carlos Salguero
@@ -3,6 +3,7 @@ Changelog for Percona Toolkit
|
|||||||
v3.0.6
|
v3.0.6
|
||||||
|
|
||||||
* Fixed bug PT-225: pt-table-checksum ignores generated cols
|
* Fixed bug PT-225: pt-table-checksum ignores generated cols
|
||||||
|
* Improvement PT-221: pt-table-sync pt-table-sync support for MyRocks
|
||||||
|
|
||||||
v3.0.5 released 2017-11-20
|
v3.0.5 released 2017-11-20
|
||||||
|
|
||||||
|
|||||||
@@ -11047,6 +11047,14 @@ sub ok_to_sync {
|
|||||||
db => $dst->{db},
|
db => $dst->{db},
|
||||||
tbl => $dst->{tbl},
|
tbl => $dst->{tbl},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
if ( lc($src->{tbl_struct}->{engine}) eq 'rocksdb' && ($o->get('sync-to-master')) ) {
|
||||||
|
print STDERR "Cannot sync using --sync-to-master with $dst->{db}.$dst->{tbl} ".
|
||||||
|
"due to the limitations of the RocksDB engine.\n\n".
|
||||||
|
"More information: https://www.percona.com/doc/percona-server/LATEST/myrocks/limitations.html\n\n";
|
||||||
|
die "Process aborted.\n";
|
||||||
|
}
|
||||||
|
|
||||||
if ( !$dst_has_table ) {
|
if ( !$dst_has_table ) {
|
||||||
die "Table $dst->{db}.$dst->{tbl} does not exist on "
|
die "Table $dst->{db}.$dst->{tbl} does not exist on "
|
||||||
. $dp->as_string($dst->{dsn}) . "\n";
|
. $dp->as_string($dst->{dsn}) . "\n";
|
||||||
|
|||||||
@@ -172,6 +172,26 @@ sub load_file {
|
|||||||
$self->wait_for_slaves() unless $args{no_wait};
|
$self->wait_for_slaves() unless $args{no_wait};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
sub has_engine {
|
||||||
|
my ( $self, $host, $want_engine ) = @_;
|
||||||
|
|
||||||
|
# Get the current checksums on the host.
|
||||||
|
my $dbh = $self->get_dbh_for($host);
|
||||||
|
my $sql = "SHOW ENGINES";
|
||||||
|
my @engines = @{$dbh->selectall_arrayref($sql, {Slice => {} })};
|
||||||
|
|
||||||
|
# Diff the two sets of checksums: host to master (ref).
|
||||||
|
my $has_engine=0;
|
||||||
|
foreach my $engine ( @engines ) {
|
||||||
|
if ( $engine->{engine} =~ m/$want_engine/i ) {
|
||||||
|
$has_engine=1;
|
||||||
|
last;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return $has_engine;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
sub _use_for {
|
sub _use_for {
|
||||||
my ( $self, $server ) = @_;
|
my ( $self, $server ) = @_;
|
||||||
return "$self->{basedir}/$port_for{$server}/use";
|
return "$self->{basedir}/$port_for{$server}/use";
|
||||||
|
|||||||
102
t/pt-table-sync/pt_221.t
Normal file
102
t/pt-table-sync/pt_221.t
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
#!/usr/bin/env perl
|
||||||
|
|
||||||
|
BEGIN {
|
||||||
|
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
|
||||||
|
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
|
||||||
|
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
|
||||||
|
};
|
||||||
|
|
||||||
|
use strict;
|
||||||
|
use warnings FATAL => 'all';
|
||||||
|
use English qw(-no_match_vars);
|
||||||
|
use Test::More;
|
||||||
|
|
||||||
|
use PerconaTest;
|
||||||
|
use Sandbox;
|
||||||
|
require "$trunk/bin/pt-table-sync";
|
||||||
|
|
||||||
|
sub set_binlog_format {
|
||||||
|
my ($sb, $format) = @_;
|
||||||
|
|
||||||
|
my $master_dbh = $sb->get_dbh_for('master');
|
||||||
|
my $slave1_dbh = $sb->get_dbh_for('slave1');
|
||||||
|
my $slave2_dbh = $sb->get_dbh_for('slave2');
|
||||||
|
|
||||||
|
$slave2_dbh->do("STOP SLAVE");
|
||||||
|
$slave1_dbh->do("STOP SLAVE");
|
||||||
|
|
||||||
|
$slave2_dbh->do("SET GLOBAL binlog_format='$format'");
|
||||||
|
$slave1_dbh->do("SET GLOBAL binlog_format='$format'");
|
||||||
|
$master_dbh->do("SET GLOBAL binlog_format='$format'");
|
||||||
|
|
||||||
|
$slave2_dbh->do("START SLAVE");
|
||||||
|
$slave1_dbh->do("START SLAVE");
|
||||||
|
}
|
||||||
|
|
||||||
|
my $dp = new DSNParser(opts=>$dsn_opts);
|
||||||
|
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp, env => q/BINLOG_FORMAT="ROW"/);
|
||||||
|
|
||||||
|
my $master_dbh = $sb->get_dbh_for('master');
|
||||||
|
my $master_dsn = $sb->dsn_for('master');
|
||||||
|
my $slave_dsn = $sb->dsn_for('slave1');
|
||||||
|
|
||||||
|
|
||||||
|
if ( !$master_dbh ) {
|
||||||
|
plan skip_all => 'Cannot connect to sandbox master';
|
||||||
|
} elsif ($sb->has_engine('master', 'rocksdb') != 1) {
|
||||||
|
plan skip_all => 'This test needs RocksDB engine';
|
||||||
|
} else {
|
||||||
|
plan tests => 4;
|
||||||
|
}
|
||||||
|
|
||||||
|
set_binlog_format($sb, 'ROW');
|
||||||
|
|
||||||
|
$master_dbh->disconnect();
|
||||||
|
$master_dbh = $sb->get_dbh_for('master');
|
||||||
|
|
||||||
|
$sb->load_file('master', 't/pt-table-sync/samples/pt_221.sql');
|
||||||
|
|
||||||
|
my @args = ('--sync-to-master', $slave_dsn, qw(-t test.t1 --print --execute));
|
||||||
|
|
||||||
|
my ($output, $exit) = full_output(
|
||||||
|
sub { pt_table_sync::main(@args, qw()) },
|
||||||
|
stderr => 1,
|
||||||
|
);
|
||||||
|
|
||||||
|
isnt(
|
||||||
|
$exit,
|
||||||
|
0,
|
||||||
|
"PT-221 fails if using --sync-to-master with RocksDB",
|
||||||
|
);
|
||||||
|
|
||||||
|
like(
|
||||||
|
$output,
|
||||||
|
qr/Cannot sync using --sync-to-master with test.t1 due to the limitations of the RocksDB engine/,
|
||||||
|
"PT-221 Cannot use --sync-to-master with RockSDB",
|
||||||
|
);
|
||||||
|
|
||||||
|
$sb->wait_for_slaves();
|
||||||
|
|
||||||
|
@args = ('--replicate', 'test.checksums', $master_dsn, qw(-t test.t1 --print --execute));
|
||||||
|
|
||||||
|
($output, $exit) = full_output(
|
||||||
|
sub { pt_table_sync::main(@args, qw()) },
|
||||||
|
stderr => 1,
|
||||||
|
);
|
||||||
|
|
||||||
|
is(
|
||||||
|
$exit,
|
||||||
|
0,
|
||||||
|
"PT-221 Doesn't fail if using --replicate with RocksDB",
|
||||||
|
);
|
||||||
|
|
||||||
|
set_binlog_format($sb, 'STATEMENT');
|
||||||
|
|
||||||
|
# #############################################################################
|
||||||
|
# Done.
|
||||||
|
# #############################################################################
|
||||||
|
$sb->wipe_clean($master_dbh);
|
||||||
|
|
||||||
|
|
||||||
|
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
|
||||||
|
exit;
|
||||||
135
t/pt-table-sync/samples/pt_221.sql
Normal file
135
t/pt-table-sync/samples/pt_221.sql
Normal file
@@ -0,0 +1,135 @@
|
|||||||
|
DROP DATABASE IF EXISTS test;
|
||||||
|
CREATE DATABASE test;
|
||||||
|
USE test;
|
||||||
|
|
||||||
|
CREATE TABLE `test`.`t1` (
|
||||||
|
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||||
|
f2 VARCHAR(25) NOT NULL,
|
||||||
|
f3 INT,
|
||||||
|
f4 timestamp
|
||||||
|
) Engine=RocksDB;
|
||||||
|
|
||||||
|
INSERT INTO `t1` VALUES
|
||||||
|
(1,'Henry Stanley',583532949,'2017-02-07 12:12:54'),
|
||||||
|
(2,'Kathy Franklin',960400044,'2017-10-10 18:06:57'),
|
||||||
|
(3,'Mr. Dr. Anthony Howard',174461349,'2017-11-16 06:06:11'),
|
||||||
|
(4,'Janice Foster I II III IV',443038240,'2016-12-07 13:01:35'),
|
||||||
|
(5,'Sharon Daniels',169641627,'2017-01-09 15:03:40'),
|
||||||
|
(6,'Alice Tucker',1012579941,'2017-06-16 06:06:28'),
|
||||||
|
(7,'Theresa Allen',894954174,'2017-07-17 10:10:00'),
|
||||||
|
(8,'Elizabeth Fowler',1938641144,'2017-03-27 14:02:24'),
|
||||||
|
(9,'Rebecca Lee',1279764421,'2017-03-06 10:10:08'),
|
||||||
|
(10,'Anne Kennedy I II III IV ',159121576,'2017-08-15 09:09:16'),
|
||||||
|
(11,'Christina Arnold',1113045364,'2017-08-18 19:07:02'),
|
||||||
|
(12,'Mrs. Ms. Miss Cheryl Garc',278843827,'2017-07-23 00:12:00'),
|
||||||
|
(13,'Louis Harvey',161885407,'2016-12-23 08:08:21'),
|
||||||
|
(14,'Julie Bishop',1592388661,'2016-11-28 01:01:03'),
|
||||||
|
(15,'Todd Ramirez',362850921,'2017-02-20 04:04:26'),
|
||||||
|
(16,'Gary Kennedy',1795393312,'2017-03-25 23:11:43'),
|
||||||
|
(17,'Roger Hart',1400528216,'2017-04-24 10:10:17'),
|
||||||
|
(18,'Lois Henry',1104948011,'2017-08-29 10:10:25'),
|
||||||
|
(19,'Debra Stanley',1103860241,'2017-08-30 16:04:47'),
|
||||||
|
(20,'Amy Cook',2001779440,'2017-03-04 14:02:36'),
|
||||||
|
(21,'Rose Vasquez I II III IV ',1025398331,'2017-01-15 05:05:04'),
|
||||||
|
(22,'David Flores',540200823,'2017-05-13 09:09:48'),
|
||||||
|
(23,'Brian Crawford',1617053422,'2017-10-14 07:07:47'),
|
||||||
|
(24,'Kelly Freeman',1361426130,'2017-04-01 04:04:17'),
|
||||||
|
(25,'Sarah Palmer',1139691470,'2017-10-23 18:06:49'),
|
||||||
|
(26,'Dorothy Shaw',900404079,'2017-10-17 10:10:01'),
|
||||||
|
(27,'Kelly Wagner',1982163577,'2016-12-01 04:04:37'),
|
||||||
|
(28,'Diane Hall',369511081,'2017-07-28 01:01:18'),
|
||||||
|
(29,'Mr. Dr. Harold Dean',811105176,'2017-11-19 13:01:31'),
|
||||||
|
(30,'Edward Ferguson',1914574524,'2017-10-14 03:03:49'),
|
||||||
|
(31,'Dorothy Arnold',1899568350,'2017-08-29 14:02:41'),
|
||||||
|
(32,'Judith Cox',17691206,'2016-12-11 04:04:20'),
|
||||||
|
(33,'Elizabeth Johnson',1685990258,'2017-06-22 20:08:50'),
|
||||||
|
(34,'Carolyn Edwards',1168962838,'2017-04-08 18:06:00'),
|
||||||
|
(35,'Ruby Chapman',1688473058,'2017-04-16 04:04:24'),
|
||||||
|
(36,'Amanda Welch I II III IV ',193262041,'2017-04-13 14:02:18'),
|
||||||
|
(37,'Larry Porter Jr. Sr. I II',75666836,'2017-04-16 16:04:09'),
|
||||||
|
(38,'Kevin George',887309316,'2017-06-13 12:12:47'),
|
||||||
|
(39,'Maria Brown',1751136810,'2017-06-08 17:05:37'),
|
||||||
|
(40,'Tammy Ramos',1588294845,'2017-07-22 11:11:24'),
|
||||||
|
(41,'Denise Ellis',1545291805,'2017-08-20 10:10:45'),
|
||||||
|
(42,'Kelly Williamson',861653714,'2017-10-10 03:03:38'),
|
||||||
|
(43,'Carolyn Patterson',2114635136,'2017-07-03 16:04:20'),
|
||||||
|
(44,'Amanda Sullivan',505888065,'2017-04-09 18:06:10'),
|
||||||
|
(45,'Scott White',2053562943,'2017-04-16 02:02:03'),
|
||||||
|
(46,'Mrs. Ms. Miss Janet Richa',273248685,'2017-01-23 14:02:17'),
|
||||||
|
(47,'Henry Davis Jr. Sr. I II ',1807469649,'2017-07-20 22:10:52'),
|
||||||
|
(48,'Sandra Powell',425895561,'2017-05-16 06:06:02'),
|
||||||
|
(49,'Roger Morrison',519405919,'2017-07-25 20:08:28'),
|
||||||
|
(50,'Henry Murray Jr. Sr. I II',2095628347,'2017-09-09 02:02:44'),
|
||||||
|
(51,'Bonnie Wood',632299815,'2017-01-21 09:09:43'),
|
||||||
|
(52,'Kevin Riley',1337811955,'2017-09-01 00:12:01'),
|
||||||
|
(53,'Cheryl Lewis',1183760979,'2017-07-10 04:04:41'),
|
||||||
|
(54,'Craig Oliver',415886351,'2017-08-27 14:02:11'),
|
||||||
|
(55,'Peter Elliott',1441905738,'2017-01-25 16:04:06'),
|
||||||
|
(56,'Sarah Henderson',1398808230,'2017-09-14 18:06:22'),
|
||||||
|
(57,'Angela Olson',73529235,'2017-02-11 19:07:04'),
|
||||||
|
(58,'Martin Hunt',828130434,'2017-04-11 23:11:01'),
|
||||||
|
(59,'Ann Grant I II III IV V M',960124334,'2017-08-01 22:10:52'),
|
||||||
|
(60,'Joe Hudson',1729961064,'2017-10-10 13:01:00'),
|
||||||
|
(61,'Mrs. Ms. Miss Shirley Rod',2124381773,'2017-05-27 20:08:23'),
|
||||||
|
(62,'Stephen Long',374271961,'2017-09-08 12:12:02'),
|
||||||
|
(63,'Gerald Butler',165044727,'2017-10-23 00:12:06'),
|
||||||
|
(64,'Tammy Hernandez',805042592,'2016-12-17 06:06:23'),
|
||||||
|
(65,'Marie Armstrong I II III ',342177736,'2017-03-16 01:01:52'),
|
||||||
|
(66,'Michelle Perry',1879564383,'2017-09-28 15:03:25'),
|
||||||
|
(67,'Albert Stevens Jr. Sr. I ',294854430,'2016-11-30 00:12:59'),
|
||||||
|
(68,'Jason Tucker',725774075,'2016-11-28 00:12:17'),
|
||||||
|
(69,'Philip Parker',482904842,'2017-08-13 06:06:22'),
|
||||||
|
(70,'Heather Stone',651794709,'2017-02-27 18:06:37'),
|
||||||
|
(71,'Mr. Dr. Keith Stone',581063673,'2017-10-31 21:09:54'),
|
||||||
|
(72,'Joseph Hicks',776452583,'2017-07-17 20:08:21'),
|
||||||
|
(73,'Judith Martinez',2056678762,'2017-04-01 06:06:00'),
|
||||||
|
(74,'Roger Fields',1025075771,'2017-09-11 03:03:03'),
|
||||||
|
(75,'Russell Alvarez',2058724771,'2017-08-28 21:09:05'),
|
||||||
|
(76,'Judith Shaw',439257644,'2016-12-06 09:09:25'),
|
||||||
|
(77,'Louis Johnston Jr. Sr. I ',1887594525,'2017-06-24 19:07:06'),
|
||||||
|
(78,'Richard Butler',39227130,'2017-07-05 16:04:22'),
|
||||||
|
(79,'Wayne Watkins Jr. Sr. I I',503442129,'2017-01-24 14:02:10'),
|
||||||
|
(80,'Patricia Stewart',286838439,'2017-11-16 06:06:24'),
|
||||||
|
(81,'Kelly Oliver',1069073876,'2017-02-24 02:02:17'),
|
||||||
|
(82,'Mrs. Ms. Miss Phyllis Fre',1053443872,'2017-05-31 04:04:19'),
|
||||||
|
(83,'Wanda Spencer',1685359738,'2017-09-27 08:08:33'),
|
||||||
|
(84,'William Harris Jr. Sr. I ',1117708440,'2017-07-24 03:03:22'),
|
||||||
|
(85,'Eugene Sanders',1605154172,'2017-09-29 13:01:06'),
|
||||||
|
(86,'Peter Sims Jr. Sr. I II I',213620615,'2017-09-06 18:06:19'),
|
||||||
|
(87,'Melissa Riley',923001454,'2017-09-30 10:10:47'),
|
||||||
|
(88,'Mr. Dr. Jonathan Lee',137651089,'2017-07-29 07:07:59'),
|
||||||
|
(89,'Donald Burke',1268762182,'2017-08-30 15:03:35'),
|
||||||
|
(90,'Adam Long',971993341,'2017-10-27 23:11:26'),
|
||||||
|
(91,'Theresa Taylor',20073496,'2017-07-26 21:09:23'),
|
||||||
|
(92,'Heather Matthews',910759132,'2017-02-19 19:07:58'),
|
||||||
|
(93,'Jerry King Jr. Sr. I II I',2055915105,'2017-06-25 15:03:29'),
|
||||||
|
(94,'Sara Hawkins',1178751968,'2017-03-29 22:10:59'),
|
||||||
|
(95,'Ruby Long',998718314,'2017-07-29 16:04:18'),
|
||||||
|
(96,'Billy Cooper',190019474,'2017-02-04 12:12:41'),
|
||||||
|
(97,'Jose Wheeler',1922987914,'2017-11-02 21:09:08'),
|
||||||
|
(98,'Clarence Weaver',364986423,'2017-09-10 00:12:31'),
|
||||||
|
(99,'Gary Riley',524227310,'2017-09-03 16:04:16'),
|
||||||
|
(100,'Melissa Parker',1833910235,'2017-10-03 05:05:26');
|
||||||
|
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS test.checksums;
|
||||||
|
|
||||||
|
CREATE TABLE test.checksums (
|
||||||
|
db CHAR(64) NOT NULL,
|
||||||
|
tbl CHAR(64) NOT NULL,
|
||||||
|
chunk INT NOT NULL,
|
||||||
|
chunk_time FLOAT NULL,
|
||||||
|
chunk_index VARCHAR(200) NULL,
|
||||||
|
lower_boundary TEXT NULL,
|
||||||
|
upper_boundary TEXT NULL,
|
||||||
|
this_crc CHAR(40) NOT NULL,
|
||||||
|
this_cnt INT NOT NULL,
|
||||||
|
master_crc CHAR(40) NULL,
|
||||||
|
master_cnt INT NULL,
|
||||||
|
ts TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||||
|
PRIMARY KEY (db, tbl, chunk),
|
||||||
|
INDEX ts_db_tbl (ts, db, tbl)
|
||||||
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
|
||||||
|
|
||||||
|
|
||||||
|
INSERT INTO test.checksums VALUES('test', 't1', 1,0,'','','100', '', 0,'',0,NOW());
|
||||||
Reference in New Issue
Block a user