Remove pt-table-sync/issue_79.t (it was broken but coincidentally working) and write better tests in filters.t. Add subs to Sandbox and PerconaTest to clear and parse tables used from the general logs. Set log=genlog in the sandbox configs.

2025-09-11 05:29:30 +00:00 · 2012-06-07 12:38:45 -04:00
parent 8d4354090e
commit 79c716d358
9 changed files with 203 additions and 175 deletions
--- a/t/lib/samples/SchemaIterator.sql
+++ b/t/lib/samples/SchemaIterator.sql
@@ -1,24 +1,43 @@
+--
+-- d1
+--
 DROP DATABASE IF EXISTS d1;
-DROP DATABASE IF EXISTS d2;
-DROP DATABASE IF EXISTS d3;
-
 CREATE DATABASE d1;
-CREATE DATABASE d2;
-CREATE DATABASE d3;

 CREATE TABLE d1.t1 (
-   i int
+  id int auto_increment primary key,
+  c  char(8)
 ) ENGINE=MyISAM CHARSET=utf8;
-INSERT INTO d1.t1 VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9);
+
 CREATE TABLE d1.t2 (
-   c varchar(255)
+  id int auto_increment primary key,
+  c  varchar(8)
 ) ENGINE=InnoDB;
+
 CREATE TABLE d1.t3 (
-   i int
+  id int auto_increment primary key,
+  c  char(8)
 ) ENGINE=MEMORY;

-CREATE TABLE d2.t1 (
-   i int
-);
+INSERT INTO d1.t1 VALUES (null, 'a'), (null, 'b'), (null, 'c'), (null, 'd'), (null, 'e'), (null, 'f'), (null, 'g'), (null, 'h'), (null, 'i');
+INSERT INTO d1.t2 SELECT * FROM d1.t1;
+INSERT INTO d1.t3 SELECT * FROM d1.t1;

+--
+-- d2
+--
+DROP DATABASE IF EXISTS d2;
+CREATE DATABASE d2;
+
+CREATE TABLE d2.t1 (
+  id int auto_increment primary key,
+  c  char(8)
+);
+-- d2.t1 is an empty table.
+
+--
+-- d3
+--
+DROP DATABASE IF EXISTS d3;
+CREATE DATABASE d3;
 -- d3 is an empty database.
--- a/t/pt-table-sync/filters.t
+++ b/t/pt-table-sync/filters.t
@@ -10,12 +10,12 @@ use strict;
 use warnings FATAL => 'all';
 use English qw(-no_match_vars);
 use Test::More;
+use Data::Dumper;

 use PerconaTest;
 use Sandbox;
 require "$trunk/bin/pt-table-sync";

-
 my $vp = new VersionParser();
 my $dp = new DSNParser(opts=>$dsn_opts);
 my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
@@ -29,81 +29,145 @@ elsif ( !$slave_dbh ) {
   plan skip_all => 'Cannot connect to sandbox slave';
 }
 else {
-   plan tests => 9;
+   plan tests => 18;
 }

-# Previous tests slave 12347 to 12346 which makes pt-table-checksum
-# complain that it cannot connect to 12347 for checking repl filters
-# and such.  12347 isn't present but SHOW SLAVE HOSTS on 12346 hasn't
-# figured that out yet, so we restart 12346 to refresh this list.
-#diag(`/tmp/12346/stop >/dev/null`);
-#diag(`/tmp/12346/start >/dev/null`);
-$slave_dbh  = $sb->get_dbh_for('slave1');
-
+my $master_dsn = $sb->dsn_for('master');
+my $slave1_dsn = $sb->dsn_for('slave1');
 my $output;
-my $cnf = "/tmp/12345/my.sandbox.cnf";
-my $cmd = "$trunk/bin/pt-table-sync -F $cnf"; 
-my $t   = qr/\d\d:\d\d:\d\d/;

-$sb->wipe_clean($master_dbh);
-$sb->load_file('master', 't/pt-table-sync/samples/filter_tables.sql');
+# See this SQL file because it has a number of simple dbs and tbls
+# that are used to checking schema object filters.
+$sb->load_file('master', "t/lib/samples/SchemaIterator.sql");

-$output = `$cmd h=127.1,P=12345 P=12346 --no-check-slave --dry-run -t issue_806_1.t2 | tail -n 2`;
-$output =~ s/$t/00:00:00/g;
-$output =~ s/[ ]{2,}/ /g;
-is(
-   $output,
-"# DELETE REPLACE INSERT UPDATE ALGORITHM START END EXIT DATABASE.TABLE
-# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_1.t2
-",
-   "db-qualified --tables (issue 806)"
+sub test_filters {
+   my (%args) = @_;
+
+   $sb->clear_genlogs();
+
+   my $output = output(
+      sub { pt_table_sync::main(@{$args{cmds}},
+         qw(--print))
+      },
+   );
+
+   my $tables_used = PerconaTest::tables_used($sb->genlog('master'));
+   is_deeply(
+      $tables_used,
+      $args{res},
+      "$args{name}: tables used"
+   ) or diag(Dumper($tables_used));
+}
+
+# #############################################################################
+# Basic schema object filters: --databases, --tables, etc.
+# #############################################################################
+
+# Not really a filter, but only the specified tables should be used.
+test_filters(
+   name => "Sync d1.t1 to d1.t2",
+   cmds => ["$master_dsn,D=d1,t=t1", "t=t2"],
+   res  => [qw(d1.t1 d1.t2)],
+);
+
+# Use slave1 like it's another master, ok becuase we're not actually
+# syncing anything, so --no-check-slave is required else pt-table-sync
+# won't run (because it doesn't like to sync directly to a slave).
+test_filters(
+   name => "-t d1.t1",
+   cmds => [$master_dsn, $slave1_dsn, qw(--no-check-slave),
+            qw(-t d1.t1)],
+   res  => [qw(d1.t1)],
+);
+
+# Like the previous test, but now any table called "t1".
+test_filters(
+   name => "-t t1",
+   cmds => [$master_dsn, $slave1_dsn, qw(--no-check-slave),
+            qw(-t t1)],
+   res  => [qw(d1.t1 d2.t1)],
+);
+
+# Only the given db, all its tables: there's only 1 tbl in d2.
+test_filters(
+   name => "--databases d2",
+   cmds => [$master_dsn, $slave1_dsn, qw(--no-check-slave),
+            qw(--databases d2)],
+   res  => [qw(d2.t1)],
+);
+
+# --database with --tables.
+test_filters(
+   name => "--databases d1 --tables t2,t3",
+   cmds => [$master_dsn, $slave1_dsn, qw(--no-check-slave),
+            qw(--databases d1), "--tables", "t2,t3"],
+   res  => [qw(d1.t2 d1.t3)],
 );

 # #############################################################################
-# Issue 820: Make mk-table-sync honor schema filters with --replicate
+# Filters with --replicate and --sync-to-master.
 # #############################################################################
-$master_dbh->do('DROP DATABASE IF EXISTS test');
-$master_dbh->do('CREATE DATABASE test');
+   
+# Checksum the filter tables.
+$master_dbh->do("DROP DATABASE IF EXISTS percona");
+$sb->wait_for_slaves();
+diag(`$trunk/bin/pt-table-checksum $master_dsn -d d1,d2,d3 --chunk-size 100 --quiet --lock-wait-timeout 3 --max-load ''`);

-$slave_dbh->do('insert into issue_806_1.t1 values (41)');
-$slave_dbh->do('insert into issue_806_2.t2 values (42)');
+my $rows = $master_dbh->selectall_arrayref("SELECT CONCAT(db, '.', tbl) FROM percona.checksums ORDER BY db, tbl");
+is_deeply(
+   $rows,
+   [
+      ["d1.t1"],
+      ["d1.t2"],
+      ["d1.t3"],
+      ["d2.t1"],
+   ],
+   "Checksummed all tables"
+) or diag(Dumper($rows));

-my $mk_table_checksum = "$trunk/bin/pt-table-checksum --lock-wait-time 3";
+# Make all checksums on the slave different than the master
+# so that pt-table-sync would sync all the tables if there
+# were no filters.
+$slave_dbh->do("UPDATE percona.checksums SET this_cnt=999 WHERE 1=1");

-`$mk_table_checksum --replicate test.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d issue_806_1,issue_806_2 --quiet`;
-
-$output = `$cmd h=127.1,P=12345 --replicate test.checksum --dry-run | tail -n 2`;
-$output =~ s/$t/00:00:00/g;
-$output =~ s/[ ]{2,}/ /g;
-is(
-   $output,
-"# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_1.t1
-# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_2.t2
-",
-   "--replicate with no filters"
+# Verify that that ^ is true.
+test_filters(
+   name => "All tables are different on the slave",
+   cmds => [$master_dsn, qw(--replicate percona.checksums)],
+   res  => [qw(d1.t1 d1.t2 d1.t3 d2.t1 percona.checksums)],
 );

-$output = `$cmd h=127.1,P=12345 --replicate test.checksum --dry-run -t t1 | tail -n 2`;
-$output =~ s/$t/00:00:00/g;
-$output =~ s/[ ]{2,}/ /g;
-is(
-   $output,
-"# DELETE REPLACE INSERT UPDATE ALGORITHM START END EXIT DATABASE.TABLE
-# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_1.t1
-",
-   "--replicate with --tables"
-);
+# Sync with --replicate, --sync-to-master, and some filters.
+# --replicate and --sync-to-master have different code paths,
+# but the filter results should be the same.
+foreach my $args (
+   [$master_dsn, qw(--replicate percona.checksums)],
+   [$slave1_dsn, qw(--replicate percona.checksums --sync-to-master)]
+) {

-$output = `$cmd h=127.1,P=12345 --replicate test.checksum --dry-run -d issue_806_2 | tail -n 2`;
-$output =~ s/$t/00:00:00/g;
-$output =~ s/[ ]{2,}/ /g;
-is(
-   $output,
-"# DELETE REPLACE INSERT UPDATE ALGORITHM START END EXIT DATABASE.TABLE
-# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_2.t2
-",
-   "--replicate with --databases"
-);
+   my $stm = $args->[-1] eq '--sync-to-master' ? ' --sync-to-master' : '';
+
+   test_filters(
+      name => $stm . "--replicate --tables t1",
+      cmds => [@$args,
+               qw(--tables t1)],
+      res  => [qw(d1.t1 d2.t1 percona.checksums)],
+   );
+
+   test_filters(
+      name => $stm . "--replicate --databases d2",
+      cmds => [@$args,
+               qw(--databases d2)],
+      res  => [qw(d2.t1 percona.checksums)],
+   );
+
+   test_filters(
+      name => $stm . "--replicate --databases d1 --tables t1,t3",
+      cmds => [@$args,
+               qw(--databases d1), "--tables", "t1,t3"],
+      res  => [qw(d1.t1 d1.t3 percona.checksums)],
+   );
+}

 # #############################################################################
 # pt-table-sync --ignore-* options don't work with --replicate 
@@ -120,7 +184,7 @@ $slave_dbh->do("INSERT INTO test.empty_it VALUES (null,11,11,'eleven')");
 diag(`$trunk/bin/pt-table-checksum h=127.1,P=12345,u=msandbox,p=msandbox -d test --quiet --quiet --lock-wait-timeout 3 --max-load ''`);

 # Make sure all the tables were checksummed.
-my $rows = $master_dbh->selectall_arrayref("SELECT DISTINCT db, tbl FROM percona.checksums ORDER BY db, tbl");
+$rows = $master_dbh->selectall_arrayref("SELECT DISTINCT db, tbl FROM percona.checksums ORDER BY db, tbl");
 is_deeply(
   $rows,
   [ [qw(test empty_it) ],
--- a/t/pt-table-sync/issue_79.t
+++ b/t/pt-table-sync/issue_79.t
@@ -1,85 +0,0 @@
-#!/usr/bin/env perl
-
-BEGIN {
-   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
-      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
-   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
-};
-
-use strict;
-use warnings FATAL => 'all';
-use English qw(-no_match_vars);
-use Test::More;
-
-use PerconaTest;
-use Sandbox;
-require "$trunk/bin/pt-table-sync";
-
-my $output;
-my $dp = new DSNParser(opts=>$dsn_opts);
-my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
-my $master_dbh = $sb->get_dbh_for('master');
-my $slave_dbh  = $sb->get_dbh_for('slave1');
-
-if ( !$master_dbh ) {
-   plan skip_all => 'Cannot connect to sandbox master';
-}
-elsif ( !$slave_dbh ) {
-   plan skip_all => 'Cannot connect to sandbox slave';
-}
-else {
-   plan tests => 7;
-}
-
-$sb->wipe_clean($master_dbh);
-$sb->create_dbs($master_dbh, [qw(test)]);
-$sb->load_file('master', 't/pt-table-sync/samples/before.sql');
-PerconaTest::wait_for_table($slave_dbh, "test.test4", "id=15034");
-
-# #############################################################################
-# Issue 79: mk-table-sync with --replicate doesn't honor --tables
-# #############################################################################
-
-$master_dbh->do('create table test.messages (i int)');
-PerconaTest::wait_for_table($slave_dbh, "test.messages", "i=3");
-$slave_dbh->do('insert into test.messages values (1), (2), (3)');
-
-# The previous test should have left test.messages on the slave (12346)
-# out of sync. Now we also unsync test2 on the slave and then re-sync only
-# it. If --tables is honored, only test2 on the slave will be synced.
-$sb->use('master', "-D test -e \"SET SQL_LOG_BIN=0; INSERT INTO test2 VALUES (1,'a'),(2,'b')\"");
-diag(`$trunk/bin/pt-table-checksum --replicate=test.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d test --lock-wait-time 3 > /dev/null`);
-
-is_deeply(
-   $master_dbh->selectall_arrayref('select * from test.messages'),
-   [],
-   'test.messages on master empty'
-);
-is_deeply(
-   $slave_dbh->selectall_arrayref('select * from test.messages'),
-   [[1],[2],[3]],
-   'test.messages on slave has values'
-);
-
-# Test that what the doc says about --tables is true:
-# "Table names may be qualified with the database name."
-# In the code, a qualified db.tbl name is used.
-# So we'll test first an unqualified tbl name.
-$output = `$trunk/bin/pt-table-sync h=127.1,P=12345,u=msandbox,p=msandbox --replicate test.checksum --execute -d test -t test2 -v`;
-unlike($output, qr/messages/, '--replicate honors --tables (1/4)');
-like($output,   qr/test2/,    '--replicate honors --tables (2/4)');
-
-# Now we'll test with a qualified db.tbl name.
-$sb->use('slave1', '-D test -e "TRUNCATE TABLE test2; TRUNCATE TABLE messages"');
-diag(`$trunk/bin/pt-table-checksum --replicate=test.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d test --lock-wait-time 3 > /dev/null`);
-
-$output = `$trunk/bin/pt-table-sync h=127.1,P=12345,u=msandbox,p=msandbox --replicate test.checksum --execute -d test -t test.test2 -v`;
-unlike($output, qr/messages/, '--replicate honors --tables (3/4)');
-like($output,   qr/test2/,    '--replicate honors --tables (4/4)');
-
-# #############################################################################
-# Done.
-# #############################################################################
-$sb->wipe_clean($master_dbh);
-ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
-exit;
--- a/t/pt-table-sync/samples/filter_tables.sql
+++ b/t/pt-table-sync/samples/filter_tables.sql
@@ -1,13 +0,0 @@
-drop database if exists issue_806_1;
-drop database if exists issue_806_2;
-create database issue_806_1;
-create database issue_806_2;
-
-use issue_806_1;
-create table t1 (i int, unique index (i));
-create table t2 (i int, unique index (i));
-
-use issue_806_2;
-create table t1 (i int, unique index (i));
-create table t2 (i int, unique index (i));
-create table t3 (i int, unique index (i));