PT-221 pt-table-sync support for MyRocks

2025-10-18 08:24:06 +00:00 · 2017-11-30 11:36:43 -03:00
parent 596d838af0
commit af69008a10
5 changed files with 266 additions and 0 deletions
--- a/1
+++ b/1
@@ -3,6 +3,7 @@ Changelog for Percona Toolkit
 v3.0.6

 * Fixed bug     PT-225: pt-table-checksum ignores generated cols
+ * Improvement   PT-221: pt-table-sync pt-table-sync support for MyRocks

 v3.0.5 released 2017-11-20

--- a/bin/pt-table-sync
+++ b/bin/pt-table-sync
@@ -11047,6 +11047,14 @@ sub ok_to_sync {
      db  => $dst->{db},
      tbl => $dst->{tbl},
   );
+
+   if ( lc($src->{tbl_struct}->{engine}) eq 'rocksdb' && ($o->get('sync-to-master')) ) {
+       print STDERR "Cannot sync using --sync-to-master with $dst->{db}.$dst->{tbl} ".
+             "due to the limitations of the RocksDB engine.\n\n".
+             "More information: https://www.percona.com/doc/percona-server/LATEST/myrocks/limitations.html\n\n";
+       die "Process aborted.\n";
+   }
+
   if ( !$dst_has_table ) {
      die "Table $dst->{db}.$dst->{tbl} does not exist on "
         . $dp->as_string($dst->{dsn}) . "\n";
--- a/lib/Sandbox.pm
+++ b/lib/Sandbox.pm
@@ -172,6 +172,26 @@ sub load_file {
   $self->wait_for_slaves() unless $args{no_wait};
 }

+sub has_engine {
+   my ( $self, $host, $want_engine ) = @_;
+
+   # Get the current checksums on the host.
+   my $dbh = $self->get_dbh_for($host);
+   my $sql = "SHOW ENGINES";
+   my @engines = @{$dbh->selectall_arrayref($sql, {Slice => {} })};
+
+   # Diff the two sets of checksums: host to master (ref).
+   my $has_engine=0;
+   foreach my $engine ( @engines ) {
+      if ( $engine->{engine} =~ m/$want_engine/i ) {
+         $has_engine=1;
+         last;
+      }
+   }
+   return $has_engine;
+}
+
+
 sub _use_for {
   my ( $self, $server ) = @_;
   return "$self->{basedir}/$port_for{$server}/use";
--- a/t/pt-table-sync/pt_221.t
+++ b/t/pt-table-sync/pt_221.t
@@ -0,0 +1,102 @@
+#!/usr/bin/env perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use Test::More;
+
+use PerconaTest;
+use Sandbox;
+require "$trunk/bin/pt-table-sync";
+
+sub set_binlog_format {
+    my ($sb, $format) = @_;
+
+    my $master_dbh = $sb->get_dbh_for('master');
+    my $slave1_dbh = $sb->get_dbh_for('slave1');
+    my $slave2_dbh = $sb->get_dbh_for('slave2');
+    
+    $slave2_dbh->do("STOP SLAVE");
+    $slave1_dbh->do("STOP SLAVE");
+    
+    $slave2_dbh->do("SET GLOBAL binlog_format='$format'");
+    $slave1_dbh->do("SET GLOBAL binlog_format='$format'");
+    $master_dbh->do("SET GLOBAL binlog_format='$format'");
+
+    $slave2_dbh->do("START SLAVE");
+    $slave1_dbh->do("START SLAVE");
+}
+
+my $dp = new DSNParser(opts=>$dsn_opts);
+my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp, env => q/BINLOG_FORMAT="ROW"/);
+
+my $master_dbh = $sb->get_dbh_for('master');
+my $master_dsn = $sb->dsn_for('master');
+my $slave_dsn  = $sb->dsn_for('slave1');
+
+
+if ( !$master_dbh ) {
+   plan skip_all => 'Cannot connect to sandbox master';
+} elsif ($sb->has_engine('master', 'rocksdb') != 1) {
+   plan skip_all => 'This test needs RocksDB engine';
+} else {
+   plan tests => 4;
+}
+
+set_binlog_format($sb, 'ROW');
+
+$master_dbh->disconnect();
+$master_dbh = $sb->get_dbh_for('master');
+
+$sb->load_file('master', 't/pt-table-sync/samples/pt_221.sql');
+
+my @args = ('--sync-to-master', $slave_dsn, qw(-t test.t1 --print --execute));
+
+my ($output, $exit) = full_output(
+   sub { pt_table_sync::main(@args, qw()) },
+   stderr => 1,
+);
+
+isnt(
+    $exit,
+    0,
+    "PT-221 fails if using --sync-to-master with RocksDB",
+);
+
+like(
+    $output,
+    qr/Cannot sync using --sync-to-master with test.t1 due to the limitations of the RocksDB engine/,
+    "PT-221 Cannot use --sync-to-master with RockSDB",
+);
+
+$sb->wait_for_slaves();
+
+@args = ('--replicate', 'test.checksums', $master_dsn, qw(-t test.t1 --print --execute));
+
+($output, $exit) = full_output(
+   sub { pt_table_sync::main(@args, qw()) },
+   stderr => 1,
+);
+
+is(
+    $exit,
+    0,
+    "PT-221 Doesn't fail if using --replicate with RocksDB",
+);
+
+set_binlog_format($sb, 'STATEMENT');
+
+# #############################################################################
+# Done.
+# #############################################################################
+$sb->wipe_clean($master_dbh);
+
+
+ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+exit;
--- a/t/pt-table-sync/samples/pt_221.sql
+++ b/t/pt-table-sync/samples/pt_221.sql
@@ -0,0 +1,135 @@
+DROP DATABASE IF EXISTS test;
+CREATE DATABASE test;
+USE test;
+
+CREATE TABLE `test`.`t1` (
+    id INT AUTO_INCREMENT PRIMARY KEY,
+    f2 VARCHAR(25) NOT NULL,
+    f3 INT,
+    f4 timestamp
+) Engine=RocksDB;
+
+INSERT INTO `t1` VALUES 
+(1,'Henry Stanley',583532949,'2017-02-07 12:12:54'),
+(2,'Kathy Franklin',960400044,'2017-10-10 18:06:57'),
+(3,'Mr. Dr. Anthony Howard',174461349,'2017-11-16 06:06:11'),
+(4,'Janice Foster I II III IV',443038240,'2016-12-07 13:01:35'),
+(5,'Sharon Daniels',169641627,'2017-01-09 15:03:40'),
+(6,'Alice Tucker',1012579941,'2017-06-16 06:06:28'),
+(7,'Theresa Allen',894954174,'2017-07-17 10:10:00'),
+(8,'Elizabeth Fowler',1938641144,'2017-03-27 14:02:24'),
+(9,'Rebecca Lee',1279764421,'2017-03-06 10:10:08'),
+(10,'Anne Kennedy I II III IV ',159121576,'2017-08-15 09:09:16'),
+(11,'Christina Arnold',1113045364,'2017-08-18 19:07:02'),
+(12,'Mrs. Ms. Miss Cheryl Garc',278843827,'2017-07-23 00:12:00'),
+(13,'Louis Harvey',161885407,'2016-12-23 08:08:21'),
+(14,'Julie Bishop',1592388661,'2016-11-28 01:01:03'),
+(15,'Todd Ramirez',362850921,'2017-02-20 04:04:26'),
+(16,'Gary Kennedy',1795393312,'2017-03-25 23:11:43'),
+(17,'Roger Hart',1400528216,'2017-04-24 10:10:17'),
+(18,'Lois Henry',1104948011,'2017-08-29 10:10:25'),
+(19,'Debra Stanley',1103860241,'2017-08-30 16:04:47'),
+(20,'Amy Cook',2001779440,'2017-03-04 14:02:36'),
+(21,'Rose Vasquez I II III IV ',1025398331,'2017-01-15 05:05:04'),
+(22,'David Flores',540200823,'2017-05-13 09:09:48'),
+(23,'Brian Crawford',1617053422,'2017-10-14 07:07:47'),
+(24,'Kelly Freeman',1361426130,'2017-04-01 04:04:17'),
+(25,'Sarah Palmer',1139691470,'2017-10-23 18:06:49'),
+(26,'Dorothy Shaw',900404079,'2017-10-17 10:10:01'),
+(27,'Kelly Wagner',1982163577,'2016-12-01 04:04:37'),
+(28,'Diane Hall',369511081,'2017-07-28 01:01:18'),
+(29,'Mr. Dr. Harold Dean',811105176,'2017-11-19 13:01:31'),
+(30,'Edward Ferguson',1914574524,'2017-10-14 03:03:49'),
+(31,'Dorothy Arnold',1899568350,'2017-08-29 14:02:41'),
+(32,'Judith Cox',17691206,'2016-12-11 04:04:20'),
+(33,'Elizabeth Johnson',1685990258,'2017-06-22 20:08:50'),
+(34,'Carolyn Edwards',1168962838,'2017-04-08 18:06:00'),
+(35,'Ruby Chapman',1688473058,'2017-04-16 04:04:24'),
+(36,'Amanda Welch I II III IV ',193262041,'2017-04-13 14:02:18'),
+(37,'Larry Porter Jr. Sr. I II',75666836,'2017-04-16 16:04:09'),
+(38,'Kevin George',887309316,'2017-06-13 12:12:47'),
+(39,'Maria Brown',1751136810,'2017-06-08 17:05:37'),
+(40,'Tammy Ramos',1588294845,'2017-07-22 11:11:24'),
+(41,'Denise Ellis',1545291805,'2017-08-20 10:10:45'),
+(42,'Kelly Williamson',861653714,'2017-10-10 03:03:38'),
+(43,'Carolyn Patterson',2114635136,'2017-07-03 16:04:20'),
+(44,'Amanda Sullivan',505888065,'2017-04-09 18:06:10'),
+(45,'Scott White',2053562943,'2017-04-16 02:02:03'),
+(46,'Mrs. Ms. Miss Janet Richa',273248685,'2017-01-23 14:02:17'),
+(47,'Henry Davis Jr. Sr. I II ',1807469649,'2017-07-20 22:10:52'),
+(48,'Sandra Powell',425895561,'2017-05-16 06:06:02'),
+(49,'Roger Morrison',519405919,'2017-07-25 20:08:28'),
+(50,'Henry Murray Jr. Sr. I II',2095628347,'2017-09-09 02:02:44'),
+(51,'Bonnie Wood',632299815,'2017-01-21 09:09:43'),
+(52,'Kevin Riley',1337811955,'2017-09-01 00:12:01'),
+(53,'Cheryl Lewis',1183760979,'2017-07-10 04:04:41'),
+(54,'Craig Oliver',415886351,'2017-08-27 14:02:11'),
+(55,'Peter Elliott',1441905738,'2017-01-25 16:04:06'),
+(56,'Sarah Henderson',1398808230,'2017-09-14 18:06:22'),
+(57,'Angela Olson',73529235,'2017-02-11 19:07:04'),
+(58,'Martin Hunt',828130434,'2017-04-11 23:11:01'),
+(59,'Ann Grant I II III IV V M',960124334,'2017-08-01 22:10:52'),
+(60,'Joe Hudson',1729961064,'2017-10-10 13:01:00'),
+(61,'Mrs. Ms. Miss Shirley Rod',2124381773,'2017-05-27 20:08:23'),
+(62,'Stephen Long',374271961,'2017-09-08 12:12:02'),
+(63,'Gerald Butler',165044727,'2017-10-23 00:12:06'),
+(64,'Tammy Hernandez',805042592,'2016-12-17 06:06:23'),
+(65,'Marie Armstrong I II III ',342177736,'2017-03-16 01:01:52'),
+(66,'Michelle Perry',1879564383,'2017-09-28 15:03:25'),
+(67,'Albert Stevens Jr. Sr. I ',294854430,'2016-11-30 00:12:59'),
+(68,'Jason Tucker',725774075,'2016-11-28 00:12:17'),
+(69,'Philip Parker',482904842,'2017-08-13 06:06:22'),
+(70,'Heather Stone',651794709,'2017-02-27 18:06:37'),
+(71,'Mr. Dr. Keith Stone',581063673,'2017-10-31 21:09:54'),
+(72,'Joseph Hicks',776452583,'2017-07-17 20:08:21'),
+(73,'Judith Martinez',2056678762,'2017-04-01 06:06:00'),
+(74,'Roger Fields',1025075771,'2017-09-11 03:03:03'),
+(75,'Russell Alvarez',2058724771,'2017-08-28 21:09:05'),
+(76,'Judith Shaw',439257644,'2016-12-06 09:09:25'),
+(77,'Louis Johnston Jr. Sr. I ',1887594525,'2017-06-24 19:07:06'),
+(78,'Richard Butler',39227130,'2017-07-05 16:04:22'),
+(79,'Wayne Watkins Jr. Sr. I I',503442129,'2017-01-24 14:02:10'),
+(80,'Patricia Stewart',286838439,'2017-11-16 06:06:24'),
+(81,'Kelly Oliver',1069073876,'2017-02-24 02:02:17'),
+(82,'Mrs. Ms. Miss Phyllis Fre',1053443872,'2017-05-31 04:04:19'),
+(83,'Wanda Spencer',1685359738,'2017-09-27 08:08:33'),
+(84,'William Harris Jr. Sr. I ',1117708440,'2017-07-24 03:03:22'),
+(85,'Eugene Sanders',1605154172,'2017-09-29 13:01:06'),
+(86,'Peter Sims Jr. Sr. I II I',213620615,'2017-09-06 18:06:19'),
+(87,'Melissa Riley',923001454,'2017-09-30 10:10:47'),
+(88,'Mr. Dr. Jonathan Lee',137651089,'2017-07-29 07:07:59'),
+(89,'Donald Burke',1268762182,'2017-08-30 15:03:35'),
+(90,'Adam Long',971993341,'2017-10-27 23:11:26'),
+(91,'Theresa Taylor',20073496,'2017-07-26 21:09:23'),
+(92,'Heather Matthews',910759132,'2017-02-19 19:07:58'),
+(93,'Jerry King Jr. Sr. I II I',2055915105,'2017-06-25 15:03:29'),
+(94,'Sara Hawkins',1178751968,'2017-03-29 22:10:59'),
+(95,'Ruby Long',998718314,'2017-07-29 16:04:18'),
+(96,'Billy Cooper',190019474,'2017-02-04 12:12:41'),
+(97,'Jose Wheeler',1922987914,'2017-11-02 21:09:08'),
+(98,'Clarence Weaver',364986423,'2017-09-10 00:12:31'),
+(99,'Gary Riley',524227310,'2017-09-03 16:04:16'),
+(100,'Melissa Parker',1833910235,'2017-10-03 05:05:26');
+
+
+DROP TABLE IF EXISTS test.checksums;
+
+CREATE TABLE test.checksums (
+   db             CHAR(64)     NOT NULL,
+   tbl            CHAR(64)     NOT NULL,
+   chunk          INT          NOT NULL,
+   chunk_time     FLOAT            NULL,
+   chunk_index    VARCHAR(200)     NULL,
+   lower_boundary TEXT             NULL,
+   upper_boundary TEXT             NULL,
+   this_crc       CHAR(40)     NOT NULL,
+   this_cnt       INT          NOT NULL,
+   master_crc     CHAR(40)         NULL,
+   master_cnt     INT              NULL,
+   ts             TIMESTAMP    NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+   PRIMARY KEY (db, tbl, chunk),
+   INDEX ts_db_tbl (ts, db, tbl)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+
+INSERT INTO test.checksums VALUES('test', 't1', 1,0,'','','100', '', 0,'',0,NOW());