From e845898189f26755a25c971b1727e5fb0c86b06a Mon Sep 17 00:00:00 2001 From: Manjot Singh Date: Wed, 25 May 2016 08:45:56 -0700 Subject: [PATCH 1/9] Begin adding --preserve-triggers to pt-osc --- bin/pt-online-schema-change | 48 +++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/bin/pt-online-schema-change b/bin/pt-online-schema-change index 7d96752b..f34577eb 100755 --- a/bin/pt-online-schema-change +++ b/bin/pt-online-schema-change @@ -10564,22 +10564,40 @@ sub create_triggers { "$new_tbl->{name}.$new_qcol <=> OLD.$old_qcol" } @{$tbl_struct->{keys}->{$del_index}->{cols}} ); - my $delete_trigger - = "CREATE TRIGGER `${prefix}_del` AFTER DELETE ON $orig_tbl->{name} " - . "FOR EACH ROW " - . "DELETE IGNORE FROM $new_tbl->{name} " - . "WHERE $del_index_cols"; + my $delete_trigger; + my $insert_trigger; + my $update_trigger; my $qcols = join(', ', map { $q->quote($_->{new}) } @$cols); my $new_vals = join(', ', map { "NEW.".$q->quote($_->{old}) } @$cols); - my $insert_trigger - = "CREATE TRIGGER `${prefix}_ins` AFTER INSERT ON $orig_tbl->{name} " - . "FOR EACH ROW " - . "REPLACE INTO $new_tbl->{name} ($qcols) VALUES ($new_vals)"; - my $update_trigger - = "CREATE TRIGGER `${prefix}_upd` AFTER UPDATE ON $orig_tbl->{name} " - . "FOR EACH ROW " - . "REPLACE INTO $new_tbl->{name} ($qcols) VALUES ($new_vals)"; + + if ( $o->get('preserve-triggers') ) { + my $dbh = $self->{cxn}->dbh; +use Data::Dumper; +print Dumper( $dbh ); + my $orig_delete_trigger = $dbh->select("SELECT TRIGGER_SCHEMA,TRIGGER_NAME,DEFINER,ACTION_STATEMENT FROM INFORMATION_SCHEMA.TRIGGERS WHERE EVENT_MANIPULATION='DELETE' AND ACTION_TIMING='AFTER' AND TRIGGER_SCHEMA='$orig_tbl->{db}' AND EVENT_OBJECT_TABLE='$orig_tbl->{name}';"); + my $orig_insert_trigger = $dbh->selectrow_arrayref("SELECT TRIGGER_SCHEMA,TRIGGER_NAME,DEFINER,ACTION_STATEMENT FROM INFORMATION_SCHEMA.TRIGGERS WHERE EVENT_MANIPULATION='INSERT' AND ACTION_TIMING='AFTER' AND TRIGGER_SCHEMA='$orig_tbl->{db}' AND EVENT_OBJECT_TABLE='$orig_tbl->{name}';"); + my $orig_update_trigger = $dbh->selectrow_arrayref("SELECT TRIGGER_SCHEMA,TRIGGER_NAME,DEFINER,ACTION_STATEMENT FROM INFORMATION_SCHEMA.TRIGGERS WHERE EVENT_MANIPULATION='UPDATE' AND ACTION_TIMING='AFTER' AND TRIGGER_SCHEMA='$orig_tbl->{db}' AND EVENT_OBJECT_TABLE='$orig_tbl->{name}';"); + + print Dumper( $orig_delete_trigger ); + #warn "after_create_new_table: $row->[1]\n\n"; + } + else { + $delete_trigger + = "CREATE TRIGGER `${prefix}_del` AFTER DELETE ON $orig_tbl->{name} " + . "FOR EACH ROW " + . "DELETE IGNORE FROM $new_tbl->{name} " + . "WHERE $del_index_cols"; + + $insert_trigger + = "CREATE TRIGGER `${prefix}_ins` AFTER INSERT ON $orig_tbl->{name} " + . "FOR EACH ROW " + . "REPLACE INTO $new_tbl->{name} ($qcols) VALUES ($new_vals)"; + $update_trigger + = "CREATE TRIGGER `${prefix}_upd` AFTER UPDATE ON $orig_tbl->{name} " + . "FOR EACH ROW " + . "REPLACE INTO $new_tbl->{name} ($qcols) VALUES ($new_vals)"; + } my @triggers = ( ['del', $delete_trigger], @@ -11616,6 +11634,10 @@ until queries are running normally again. This will not prevent queueing, however; it will only give the server a chance to recover from the queueing. If you notice queueing, it is best to decrease the chunk time. +=item --preserve-triggers + +Preserves old triggers when specified. + =item --new-table-name type: string; default: %T_new From 67018f88f90ef1e946526459b543c49e1be2a696 Mon Sep 17 00:00:00 2001 From: Manjot Singh Date: Wed, 25 May 2016 08:51:03 -0700 Subject: [PATCH 2/9] fixed broken debug pt-osc --- bin/pt-online-schema-change | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/pt-online-schema-change b/bin/pt-online-schema-change index f34577eb..f925d7d4 100755 --- a/bin/pt-online-schema-change +++ b/bin/pt-online-schema-change @@ -10572,10 +10572,10 @@ sub create_triggers { my $new_vals = join(', ', map { "NEW.".$q->quote($_->{old}) } @$cols); if ( $o->get('preserve-triggers') ) { - my $dbh = $self->{cxn}->dbh; + my $dbh = $cxn->dbh; use Data::Dumper; print Dumper( $dbh ); - my $orig_delete_trigger = $dbh->select("SELECT TRIGGER_SCHEMA,TRIGGER_NAME,DEFINER,ACTION_STATEMENT FROM INFORMATION_SCHEMA.TRIGGERS WHERE EVENT_MANIPULATION='DELETE' AND ACTION_TIMING='AFTER' AND TRIGGER_SCHEMA='$orig_tbl->{db}' AND EVENT_OBJECT_TABLE='$orig_tbl->{name}';"); + my $orig_delete_trigger = $dbh->selectrow_arrayref("SELECT TRIGGER_SCHEMA,TRIGGER_NAME,DEFINER,ACTION_STATEMENT FROM INFORMATION_SCHEMA.TRIGGERS WHERE EVENT_MANIPULATION='DELETE' AND ACTION_TIMING='AFTER' AND TRIGGER_SCHEMA='$orig_tbl->{db}' AND EVENT_OBJECT_TABLE='$orig_tbl->{name}';"); my $orig_insert_trigger = $dbh->selectrow_arrayref("SELECT TRIGGER_SCHEMA,TRIGGER_NAME,DEFINER,ACTION_STATEMENT FROM INFORMATION_SCHEMA.TRIGGERS WHERE EVENT_MANIPULATION='INSERT' AND ACTION_TIMING='AFTER' AND TRIGGER_SCHEMA='$orig_tbl->{db}' AND EVENT_OBJECT_TABLE='$orig_tbl->{name}';"); my $orig_update_trigger = $dbh->selectrow_arrayref("SELECT TRIGGER_SCHEMA,TRIGGER_NAME,DEFINER,ACTION_STATEMENT FROM INFORMATION_SCHEMA.TRIGGERS WHERE EVENT_MANIPULATION='UPDATE' AND ACTION_TIMING='AFTER' AND TRIGGER_SCHEMA='$orig_tbl->{db}' AND EVENT_OBJECT_TABLE='$orig_tbl->{name}';"); From 19306adcd54d1e0fff26a20a7b2f062aaaaebaa8 Mon Sep 17 00:00:00 2001 From: Manjot Singh Date: Wed, 25 May 2016 20:45:13 -0700 Subject: [PATCH 3/9] Have pt-osc changes working for creating delete trigger --- bin/pt-online-schema-change | 98 +++++++++++++++++++++++++++++++++---- 1 file changed, 88 insertions(+), 10 deletions(-) diff --git a/bin/pt-online-schema-change b/bin/pt-online-schema-change index f925d7d4..851ec0a9 100755 --- a/bin/pt-online-schema-change +++ b/bin/pt-online-schema-change @@ -10210,7 +10210,7 @@ sub check_orig_table { . ' LIKE ' . $q->literal_like($orig_tbl->{tbl}); PTDEBUG && _d($sql); my $triggers = $dbh->selectall_arrayref($sql); - if ( $triggers && @$triggers ) { + if ( $triggers && @$triggers && !$o->get('preserve-triggers')) { die "The table $orig_tbl->{name} has triggers. This tool " . "needs to create its own triggers, so the table cannot " . "already have triggers.\n"; @@ -10571,16 +10571,61 @@ sub create_triggers { my $qcols = join(', ', map { $q->quote($_->{new}) } @$cols); my $new_vals = join(', ', map { "NEW.".$q->quote($_->{old}) } @$cols); + + my $orig_delete_trigger; + my $orig_insert_trigger; + my $orig_update_trigger; + if ( $o->get('preserve-triggers') ) { - my $dbh = $cxn->dbh; -use Data::Dumper; -print Dumper( $dbh ); - my $orig_delete_trigger = $dbh->selectrow_arrayref("SELECT TRIGGER_SCHEMA,TRIGGER_NAME,DEFINER,ACTION_STATEMENT FROM INFORMATION_SCHEMA.TRIGGERS WHERE EVENT_MANIPULATION='DELETE' AND ACTION_TIMING='AFTER' AND TRIGGER_SCHEMA='$orig_tbl->{db}' AND EVENT_OBJECT_TABLE='$orig_tbl->{name}';"); - my $orig_insert_trigger = $dbh->selectrow_arrayref("SELECT TRIGGER_SCHEMA,TRIGGER_NAME,DEFINER,ACTION_STATEMENT FROM INFORMATION_SCHEMA.TRIGGERS WHERE EVENT_MANIPULATION='INSERT' AND ACTION_TIMING='AFTER' AND TRIGGER_SCHEMA='$orig_tbl->{db}' AND EVENT_OBJECT_TABLE='$orig_tbl->{name}';"); - my $orig_update_trigger = $dbh->selectrow_arrayref("SELECT TRIGGER_SCHEMA,TRIGGER_NAME,DEFINER,ACTION_STATEMENT FROM INFORMATION_SCHEMA.TRIGGERS WHERE EVENT_MANIPULATION='UPDATE' AND ACTION_TIMING='AFTER' AND TRIGGER_SCHEMA='$orig_tbl->{db}' AND EVENT_OBJECT_TABLE='$orig_tbl->{name}';"); - - print Dumper( $orig_delete_trigger ); - #warn "after_create_new_table: $row->[1]\n\n"; + $cxn->connect(); + + my $dbh = $cxn->dbh(); + + $orig_delete_trigger = $dbh->selectrow_arrayref("SELECT TRIGGER_SCHEMA,TRIGGER_NAME,DEFINER,ACTION_STATEMENT FROM INFORMATION_SCHEMA.TRIGGERS WHERE EVENT_MANIPULATION='DELETE' AND ACTION_TIMING='AFTER' AND TRIGGER_SCHEMA='$orig_tbl->{db}' AND EVENT_OBJECT_TABLE='$orig_tbl->{tbl}';"); + $orig_insert_trigger = $dbh->selectrow_arrayref("SELECT TRIGGER_SCHEMA,TRIGGER_NAME,DEFINER,ACTION_STATEMENT FROM INFORMATION_SCHEMA.TRIGGERS WHERE EVENT_MANIPULATION='INSERT' AND ACTION_TIMING='AFTER' AND TRIGGER_SCHEMA='$orig_tbl->{db}' AND EVENT_OBJECT_TABLE='$orig_tbl->{tbl}';"); + $orig_update_trigger = $dbh->selectrow_arrayref("SELECT TRIGGER_SCHEMA,TRIGGER_NAME,DEFINER,ACTION_STATEMENT FROM INFORMATION_SCHEMA.TRIGGERS WHERE EVENT_MANIPULATION='UPDATE' AND ACTION_TIMING='AFTER' AND TRIGGER_SCHEMA='$orig_tbl->{db}' AND EVENT_OBJECT_TABLE='$orig_tbl->{tbl}';"); + } + + if ( $o->get('preserve-triggers') && $orig_delete_trigger) { + + # find if trigger has begin/end - supply insertion point + my $delete_trigger_ins_point = trigger_ins_point(trigger => $orig_delete_trigger->[3]); + my $insert_trigger_ins_point = trigger_ins_point(trigger => $orig_insert_trigger->[3]); + my $update_trigger_ins_point = trigger_ins_point(trigger => $orig_update_trigger->[3]); + + $delete_trigger + = "CREATE TRIGGER `${prefix}_del` AFTER DELETE ON $orig_tbl->{name} " + . "FOR EACH ROW"; + + my $delete_trigger_action + = "DELETE IGNORE FROM $new_tbl->{name} " + . "WHERE $del_index_cols"; + + if($delete_trigger_ins_point) { + $delete_trigger + = "LOCK TABLES $orig_tbl->{name};\n" + . "DROP TRIGGER `$orig_tbl->{db}`.`$orig_delete_trigger->[1]`;\n" + . "DELIMITER //\n" + . $delete_trigger . "\n" + . substr($orig_delete_trigger->[3], 0, $delete_trigger_ins_point) + . $delete_trigger_action . ";\n" + . substr($orig_delete_trigger->[3], $delete_trigger_ins_point) . "//\n" + . "DELIMITER ;\n" + . "UNLOCK TABLES;"; + } + else { + $delete_trigger + = "LOCK TABLES $orig_tbl->{name};\n" + . "DROP TRIGGER `$orig_tbl->{db}`.`$orig_delete_trigger->[1]`;\n" + . "DELIMITER //\n" + . $delete_trigger . "\n" + . "BEGIN\n" + . $orig_delete_trigger->[3] . "\n" + . $delete_trigger_action . ";\n" + . "END; //\n" + . "DELIMITER ;\n" + . "UNLOCK TABLES;"; + } } else { $delete_trigger @@ -10588,6 +10633,22 @@ print Dumper( $dbh ); . "FOR EACH ROW " . "DELETE IGNORE FROM $new_tbl->{name} " . "WHERE $del_index_cols"; + } + + if ( $o->get('preserve-triggers') && $orig_insert_trigger) { + + $insert_trigger + = "CREATE TRIGGER `${prefix}_ins` AFTER INSERT ON $orig_tbl->{name} " + . "FOR EACH ROW " + . "REPLACE INTO $new_tbl->{name} ($qcols) VALUES ($new_vals)"; + $update_trigger + = "CREATE TRIGGER `${prefix}_upd` AFTER UPDATE ON $orig_tbl->{name} " + . "FOR EACH ROW " + . "REPLACE INTO $new_tbl->{name} ($qcols) VALUES ($new_vals)"; + + + } + else { $insert_trigger = "CREATE TRIGGER `${prefix}_ins` AFTER INSERT ON $orig_tbl->{name} " @@ -10917,6 +10978,23 @@ sub ts { return $msg ? "$ts $msg" : $ts; } +sub trigger_ins_point { + my ( %args ) = @_; + my @required_args = qw(trigger); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless defined $args{$arg}; + } + my ($trigger) = @args{@required_args}; + + my $ins_point; + if ($trigger =~ /begin(.*?)end(?!.*end)/igms) { + $ins_point = $+[0] - 3; + } + else { $ins_point = 0;} + + return $ins_point; +} + # Catches signals so we can exit gracefully. sub sig_int { my ( $signal ) = @_; From 99b8e761432fc5b537124592eaf9044421089735 Mon Sep 17 00:00:00 2001 From: Manjot Singh Date: Thu, 26 May 2016 15:57:52 -0700 Subject: [PATCH 4/9] Completed logic for pt-osc preserve triggers Issue with some sql, currently working on. --- bin/pt-online-schema-change | 140 ++++++++++++++++++++++++++++++++---- 1 file changed, 126 insertions(+), 14 deletions(-) diff --git a/bin/pt-online-schema-change b/bin/pt-online-schema-change index 851ec0a9..8ac8a5dd 100755 --- a/bin/pt-online-schema-change +++ b/bin/pt-online-schema-change @@ -10576,6 +10576,12 @@ sub create_triggers { my $orig_insert_trigger; my $orig_update_trigger; + my %orig_triggers = ( + del => '', + ins => '', + upd => '' + ); + if ( $o->get('preserve-triggers') ) { $cxn->connect(); @@ -10590,8 +10596,6 @@ sub create_triggers { # find if trigger has begin/end - supply insertion point my $delete_trigger_ins_point = trigger_ins_point(trigger => $orig_delete_trigger->[3]); - my $insert_trigger_ins_point = trigger_ins_point(trigger => $orig_insert_trigger->[3]); - my $update_trigger_ins_point = trigger_ins_point(trigger => $orig_update_trigger->[3]); $delete_trigger = "CREATE TRIGGER `${prefix}_del` AFTER DELETE ON $orig_tbl->{name} " @@ -10604,19 +10608,19 @@ sub create_triggers { if($delete_trigger_ins_point) { $delete_trigger = "LOCK TABLES $orig_tbl->{name};\n" - . "DROP TRIGGER `$orig_tbl->{db}`.`$orig_delete_trigger->[1]`;\n" + . "DROP TRIGGER IF EXISTS `$orig_tbl->{db}`.`$orig_delete_trigger->[1]`;\n" . "DELIMITER //\n" . $delete_trigger . "\n" . substr($orig_delete_trigger->[3], 0, $delete_trigger_ins_point) . $delete_trigger_action . ";\n" - . substr($orig_delete_trigger->[3], $delete_trigger_ins_point) . "//\n" + . substr($orig_delete_trigger->[3], $delete_trigger_ins_point) . "; //\n" . "DELIMITER ;\n" . "UNLOCK TABLES;"; } else { $delete_trigger = "LOCK TABLES $orig_tbl->{name};\n" - . "DROP TRIGGER `$orig_tbl->{db}`.`$orig_delete_trigger->[1]`;\n" + . "DROP TRIGGER IF EXISTS `$orig_tbl->{db}`.`$orig_delete_trigger->[1]`;\n" . "DELIMITER //\n" . $delete_trigger . "\n" . "BEGIN\n" @@ -10626,6 +10630,18 @@ sub create_triggers { . "DELIMITER ;\n" . "UNLOCK TABLES;"; } + print $delete_trigger; + $orig_triggers{del} + = "LOCK TABLES $orig_tbl->{name};\n" + . "DROP TRIGGER IF EXISTS `${prefix}_del`;\n" + . "DELIMITER //\n" + . "CREATE DEFINER=$orig_delete_trigger->[2] TRIGGER `$orig_delete_trigger->[0]`.`$orig_delete_trigger->[1]` " + . "AFTER DELETE ON $orig_tbl->{name}\n" + . "FOR EACH ROW\n" + . $orig_delete_trigger->[3] . "\n" + . "//\n" + . "DELIMITER ;\n" + . "UNLOCK TABLES;"; } else { $delete_trigger @@ -10636,24 +10652,117 @@ sub create_triggers { } if ( $o->get('preserve-triggers') && $orig_insert_trigger) { - + + # find if trigger has begin/end - supply insertion point + my $insert_trigger_ins_point = trigger_ins_point(trigger => $orig_insert_trigger->[3]); + my $update_trigger_ins_point = trigger_ins_point(trigger => $orig_update_trigger->[3]); + + $insert_trigger + = "CREATE TRIGGER `${prefix}_ins` AFTER INSERT ON $orig_tbl->{name} " + . "FOR EACH ROW"; + + my $insert_trigger_action + = "REPLACE INTO $new_tbl->{name} ($qcols) VALUES ($new_vals)"; + + if($insert_trigger_ins_point) { + $insert_trigger + = "LOCK TABLES $orig_tbl->{name};\n" + . "DROP TRIGGER IF EXISTS `$orig_tbl->{db}`.`$orig_insert_trigger->[1]`;\n" + . "DELIMITER //\n" + . $insert_trigger . "\n" + . substr($orig_insert_trigger->[3], 0, $insert_trigger_ins_point) + . $insert_trigger_action . ";\n" + . substr($orig_insert_trigger->[3], $insert_trigger_ins_point) . "//\n" + . "DELIMITER ;\n" + . "UNLOCK TABLES;"; + } + else { + $insert_trigger + = "LOCK TABLES $orig_tbl->{name};\n" + . "DROP TRIGGER IF EXISTS `$orig_tbl->{db}`.`$orig_insert_trigger->[1]`;\n" + . "DELIMITER //\n" + . $insert_trigger . "\n" + . "BEGIN\n" + . $orig_insert_trigger->[3] . "\n" + . $insert_trigger_action . ";\n" + . "END; //\n" + . "DELIMITER ;\n" + . "UNLOCK TABLES;"; + } + + $orig_triggers{del} + = "LOCK TABLES $orig_tbl->{name};\n" + . "DROP TRIGGER IF EXISTS `${prefix}_del`;\n" + . "DELIMITER //\n" + . "CREATE DEFINER=$orig_insert_trigger->[2] TRIGGER `$orig_insert_trigger->[0]`.`$orig_insert_trigger->[1]` " + . "AFTER DELETE ON $orig_tbl->{name}\n" + . "FOR EACH ROW\n" + . $orig_insert_trigger->[3] . "\n" + . "//\n" + . "DELIMITER ;\n" + . "UNLOCK TABLES;"; + } + else { $insert_trigger = "CREATE TRIGGER `${prefix}_ins` AFTER INSERT ON $orig_tbl->{name} " - . "FOR EACH ROW " - . "REPLACE INTO $new_tbl->{name} ($qcols) VALUES ($new_vals)"; - $update_trigger - = "CREATE TRIGGER `${prefix}_upd` AFTER UPDATE ON $orig_tbl->{name} " . "FOR EACH ROW " . "REPLACE INTO $new_tbl->{name} ($qcols) VALUES ($new_vals)"; } + + if ( $o->get('preserve-triggers') && $orig_update_trigger) { + + # find if trigger has begin/end - supply insertion point + my $update_trigger_ins_point = trigger_ins_point(trigger => $orig_update_trigger->[3]); + + $update_trigger + = "CREATE TRIGGER `${prefix}_upd` AFTER UPDATE ON $orig_tbl->{name} " + . "FOR EACH ROW"; + + my $update_trigger_action + = "REPLACE INTO $new_tbl->{name} ($qcols) VALUES ($new_vals)"; + + if($update_trigger_ins_point) { + $update_trigger + = "LOCK TABLES $orig_tbl->{name};\n" + . "DROP TRIGGER IF EXISTS `$orig_tbl->{db}`.`$orig_update_trigger->[1]`;\n" + . "DELIMITER //\n" + . $update_trigger . "\n" + . substr($orig_update_trigger->[3], 0, $update_trigger_ins_point) + . $update_trigger_action . ";\n" + . substr($orig_update_trigger->[3], $update_trigger_ins_point) . "//\n" + . "DELIMITER ;\n" + . "UNLOCK TABLES;"; + } + else { + $update_trigger + = "LOCK TABLES $orig_tbl->{name};\n" + . "DROP TRIGGER IF EXISTS `$orig_tbl->{db}`.`$orig_update_trigger->[1]`;\n" + . "DELIMITER //\n" + . $update_trigger . "\n" + . "BEGIN\n" + . $orig_update_trigger->[3] . "\n" + . $update_trigger_action . ";\n" + . "END; //\n" + . "DELIMITER ;\n" + . "UNLOCK TABLES;"; + } + + $orig_triggers{del} + = "LOCK TABLES $orig_tbl->{name};\n" + . "DROP TRIGGER IF EXISTS `${prefix}_del`;\n" + . "DELIMITER //\n" + . "CREATE DEFINER=$orig_update_trigger->[2] TRIGGER `$orig_update_trigger->[0]`.`$orig_update_trigger->[1]` " + . "AFTER DELETE ON $orig_tbl->{name}\n" + . "FOR EACH ROW\n" + . $orig_update_trigger->[3] . "\n" + . "//\n" + . "DELIMITER ;\n" + . "UNLOCK TABLES;"; + } else { - $insert_trigger - = "CREATE TRIGGER `${prefix}_ins` AFTER INSERT ON $orig_tbl->{name} " - . "FOR EACH ROW " - . "REPLACE INTO $new_tbl->{name} ($qcols) VALUES ($new_vals)"; $update_trigger = "CREATE TRIGGER `${prefix}_upd` AFTER UPDATE ON $orig_tbl->{name} " . "FOR EACH ROW " @@ -10687,6 +10796,9 @@ sub create_triggers { # (or faked to be created) so if the 2nd trigger # fails to create, we know to only drop the 1st. push @trigger_names, "${prefix}_$name"; + if ( $o->get('preserve-triggers') ) { + push @drop_trigger_sqls, $orig_triggers{$name}; + } push @drop_trigger_sqls, "DROP TRIGGER IF EXISTS " . $q->quote($orig_tbl->{db}, "${prefix}_$name") . ";"; From 5cb5f0341771f14e6286190fd17410a9a5e293fa Mon Sep 17 00:00:00 2001 From: Manjot Singh Date: Thu, 26 May 2016 17:06:12 -0700 Subject: [PATCH 5/9] pt-osc mostly working now. Need to change original trigger logic to add to the new table before replacement. --- bin/pt-online-schema-change | 60 +++++++++++++------------------------ 1 file changed, 20 insertions(+), 40 deletions(-) diff --git a/bin/pt-online-schema-change b/bin/pt-online-schema-change index 8ac8a5dd..8b13fe97 100755 --- a/bin/pt-online-schema-change +++ b/bin/pt-online-schema-change @@ -2253,6 +2253,7 @@ sub get_dbh { PrintError => 0, ShowErrorStatement => 1, mysql_enable_utf8 => ($cxn_string =~ m/charset=utf8/i ? 1 : 0), + mysql_multi_statements => 1, }; @{$defaults}{ keys %$opts } = values %$opts; if (delete $defaults->{L}) { # L for LOAD DATA LOCAL INFILE, our own extension @@ -10607,40 +10608,33 @@ sub create_triggers { if($delete_trigger_ins_point) { $delete_trigger - = "LOCK TABLES $orig_tbl->{name};\n" + = "LOCK TABLES $orig_tbl->{name} WRITE;\n" . "DROP TRIGGER IF EXISTS `$orig_tbl->{db}`.`$orig_delete_trigger->[1]`;\n" - . "DELIMITER //\n" . $delete_trigger . "\n" . substr($orig_delete_trigger->[3], 0, $delete_trigger_ins_point) . $delete_trigger_action . ";\n" - . substr($orig_delete_trigger->[3], $delete_trigger_ins_point) . "; //\n" - . "DELIMITER ;\n" + . substr($orig_delete_trigger->[3], $delete_trigger_ins_point) . ";\n" . "UNLOCK TABLES;"; } else { $delete_trigger - = "LOCK TABLES $orig_tbl->{name};\n" + = "LOCK TABLES $orig_tbl->{name} WRITE;\n" . "DROP TRIGGER IF EXISTS `$orig_tbl->{db}`.`$orig_delete_trigger->[1]`;\n" - . "DELIMITER //\n" . $delete_trigger . "\n" . "BEGIN\n" . $orig_delete_trigger->[3] . "\n" . $delete_trigger_action . ";\n" - . "END; //\n" - . "DELIMITER ;\n" + . "END;\n" . "UNLOCK TABLES;"; } - print $delete_trigger; + $orig_triggers{del} - = "LOCK TABLES $orig_tbl->{name};\n" + = "LOCK TABLES $orig_tbl->{name} WRITE;\n" . "DROP TRIGGER IF EXISTS `${prefix}_del`;\n" - . "DELIMITER //\n" . "CREATE DEFINER=$orig_delete_trigger->[2] TRIGGER `$orig_delete_trigger->[0]`.`$orig_delete_trigger->[1]` " . "AFTER DELETE ON $orig_tbl->{name}\n" . "FOR EACH ROW\n" - . $orig_delete_trigger->[3] . "\n" - . "//\n" - . "DELIMITER ;\n" + . $orig_delete_trigger->[3] . ";\n" . "UNLOCK TABLES;"; } else { @@ -10666,40 +10660,33 @@ sub create_triggers { if($insert_trigger_ins_point) { $insert_trigger - = "LOCK TABLES $orig_tbl->{name};\n" + = "LOCK TABLES $orig_tbl->{name} WRITE;\n" . "DROP TRIGGER IF EXISTS `$orig_tbl->{db}`.`$orig_insert_trigger->[1]`;\n" - . "DELIMITER //\n" . $insert_trigger . "\n" . substr($orig_insert_trigger->[3], 0, $insert_trigger_ins_point) . $insert_trigger_action . ";\n" - . substr($orig_insert_trigger->[3], $insert_trigger_ins_point) . "//\n" - . "DELIMITER ;\n" + . substr($orig_insert_trigger->[3], $insert_trigger_ins_point) . ";\n" . "UNLOCK TABLES;"; } else { $insert_trigger - = "LOCK TABLES $orig_tbl->{name};\n" + = "LOCK TABLES $orig_tbl->{name} WRITE;\n" . "DROP TRIGGER IF EXISTS `$orig_tbl->{db}`.`$orig_insert_trigger->[1]`;\n" - . "DELIMITER //\n" . $insert_trigger . "\n" . "BEGIN\n" . $orig_insert_trigger->[3] . "\n" . $insert_trigger_action . ";\n" - . "END; //\n" - . "DELIMITER ;\n" + . "END; \n" . "UNLOCK TABLES;"; } $orig_triggers{del} - = "LOCK TABLES $orig_tbl->{name};\n" + = "LOCK TABLES $orig_tbl->{name} WRITE;\n" . "DROP TRIGGER IF EXISTS `${prefix}_del`;\n" - . "DELIMITER //\n" . "CREATE DEFINER=$orig_insert_trigger->[2] TRIGGER `$orig_insert_trigger->[0]`.`$orig_insert_trigger->[1]` " . "AFTER DELETE ON $orig_tbl->{name}\n" . "FOR EACH ROW\n" - . $orig_insert_trigger->[3] . "\n" - . "//\n" - . "DELIMITER ;\n" + . $orig_insert_trigger->[3] . ";\n" . "UNLOCK TABLES;"; } else { @@ -10725,40 +10712,33 @@ sub create_triggers { if($update_trigger_ins_point) { $update_trigger - = "LOCK TABLES $orig_tbl->{name};\n" + = "LOCK TABLES $orig_tbl->{name} WRITE;\n" . "DROP TRIGGER IF EXISTS `$orig_tbl->{db}`.`$orig_update_trigger->[1]`;\n" - . "DELIMITER //\n" . $update_trigger . "\n" . substr($orig_update_trigger->[3], 0, $update_trigger_ins_point) . $update_trigger_action . ";\n" - . substr($orig_update_trigger->[3], $update_trigger_ins_point) . "//\n" - . "DELIMITER ;\n" + . substr($orig_update_trigger->[3], $update_trigger_ins_point) . ";\n" . "UNLOCK TABLES;"; } else { $update_trigger - = "LOCK TABLES $orig_tbl->{name};\n" + = "LOCK TABLES $orig_tbl->{name} WRITE;\n" . "DROP TRIGGER IF EXISTS `$orig_tbl->{db}`.`$orig_update_trigger->[1]`;\n" - . "DELIMITER //\n" . $update_trigger . "\n" . "BEGIN\n" . $orig_update_trigger->[3] . "\n" . $update_trigger_action . ";\n" - . "END; //\n" - . "DELIMITER ;\n" + . "END;\n" . "UNLOCK TABLES;"; } $orig_triggers{del} - = "LOCK TABLES $orig_tbl->{name};\n" + = "LOCK TABLES $orig_tbl->{name} WRITE;\n" . "DROP TRIGGER IF EXISTS `${prefix}_del`;\n" - . "DELIMITER //\n" . "CREATE DEFINER=$orig_update_trigger->[2] TRIGGER `$orig_update_trigger->[0]`.`$orig_update_trigger->[1]` " . "AFTER DELETE ON $orig_tbl->{name}\n" . "FOR EACH ROW\n" - . $orig_update_trigger->[3] . "\n" - . "//\n" - . "DELIMITER ;\n" + . $orig_update_trigger->[3] . ";\n" . "UNLOCK TABLES;"; } else { From 20641dab2bf047fef5e91b20110d6fa918301c8b Mon Sep 17 00:00:00 2001 From: Manjot Singh Date: Thu, 26 May 2016 23:36:17 -0700 Subject: [PATCH 6/9] Working version with --preserve-triggers --- bin/pt-online-schema-change | 67 ++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/bin/pt-online-schema-change b/bin/pt-online-schema-change index 8b13fe97..d4ce86c3 100755 --- a/bin/pt-online-schema-change +++ b/bin/pt-online-schema-change @@ -8030,6 +8030,7 @@ my $dont_interrupt_now = 0; my @drop_trigger_sqls; my @triggers_not_dropped; my $pxc_version = '0'; +my @orig_triggers = ('','',''); # Completely ignore these error codes. my %ignore_code = ( # Error: 1592 SQLSTATE: HY000 (ER_BINLOG_UNSAFE_STATEMENT) @@ -9546,6 +9547,16 @@ sub main { $plugin->before_swap_tables(); } + if ( $o->get('preserve-triggers') ) { + print "Adding original triggers to new table.\n"; + + foreach my $orig_trigger (@orig_triggers) { + PTDEBUG && _d($orig_trigger); + print $orig_trigger, "\n" if $o->get('print'); + $cxn->dbh()->do($orig_trigger); + } + } + my $old_tbl; if ( $o->get('swap-tables') ) { @@ -10577,11 +10588,6 @@ sub create_triggers { my $orig_insert_trigger; my $orig_update_trigger; - my %orig_triggers = ( - del => '', - ins => '', - upd => '' - ); if ( $o->get('preserve-triggers') ) { $cxn->connect(); @@ -10628,14 +10634,15 @@ sub create_triggers { . "UNLOCK TABLES;"; } - $orig_triggers{del} - = "LOCK TABLES $orig_tbl->{name} WRITE;\n" - . "DROP TRIGGER IF EXISTS `${prefix}_del`;\n" - . "CREATE DEFINER=$orig_delete_trigger->[2] TRIGGER `$orig_delete_trigger->[0]`.`$orig_delete_trigger->[1]` " - . "AFTER DELETE ON $orig_tbl->{name}\n" + my $definer = $orig_delete_trigger->[2]; + $definer =~ s/@/`@`/; + $definer = "`".$definer."`"; + + $orig_triggers[0] + = "CREATE DEFINER=$definer TRIGGER `$orig_delete_trigger->[0]`.`$orig_delete_trigger->[1]` " + . "AFTER DELETE ON $new_tbl->{name}\n" . "FOR EACH ROW\n" - . $orig_delete_trigger->[3] . ";\n" - . "UNLOCK TABLES;"; + . $orig_delete_trigger->[3] . ";\n"; } else { $delete_trigger @@ -10680,14 +10687,15 @@ sub create_triggers { . "UNLOCK TABLES;"; } - $orig_triggers{del} - = "LOCK TABLES $orig_tbl->{name} WRITE;\n" - . "DROP TRIGGER IF EXISTS `${prefix}_del`;\n" - . "CREATE DEFINER=$orig_insert_trigger->[2] TRIGGER `$orig_insert_trigger->[0]`.`$orig_insert_trigger->[1]` " - . "AFTER DELETE ON $orig_tbl->{name}\n" + my $definer = $orig_insert_trigger->[2]; + $definer =~ s/@/`@`/; + $definer = "`".$definer."`"; + + $orig_triggers[1] + = "CREATE DEFINER=$definer TRIGGER `$orig_insert_trigger->[0]`.`$orig_insert_trigger->[1]` " + . "AFTER INSERT ON $new_tbl->{name}\n" . "FOR EACH ROW\n" - . $orig_insert_trigger->[3] . ";\n" - . "UNLOCK TABLES;"; + . $orig_insert_trigger->[3] . ";\n"; } else { $insert_trigger @@ -10732,14 +10740,15 @@ sub create_triggers { . "UNLOCK TABLES;"; } - $orig_triggers{del} - = "LOCK TABLES $orig_tbl->{name} WRITE;\n" - . "DROP TRIGGER IF EXISTS `${prefix}_del`;\n" - . "CREATE DEFINER=$orig_update_trigger->[2] TRIGGER `$orig_update_trigger->[0]`.`$orig_update_trigger->[1]` " - . "AFTER DELETE ON $orig_tbl->{name}\n" + my $definer = $orig_update_trigger->[2]; + $definer =~ s/@/`@`/; + $definer = "`".$definer."`"; + + $orig_triggers[2] + = "CREATE DEFINER=$definer TRIGGER `$orig_update_trigger->[0]`.`$orig_update_trigger->[1]` " + . "AFTER UPDATE ON $new_tbl->{name}\n" . "FOR EACH ROW\n" - . $orig_update_trigger->[3] . ";\n" - . "UNLOCK TABLES;"; + . $orig_update_trigger->[3] . ";\n"; } else { @@ -10776,9 +10785,6 @@ sub create_triggers { # (or faked to be created) so if the 2nd trigger # fails to create, we know to only drop the 1st. push @trigger_names, "${prefix}_$name"; - if ( $o->get('preserve-triggers') ) { - push @drop_trigger_sqls, $orig_triggers{$name}; - } push @drop_trigger_sqls, "DROP TRIGGER IF EXISTS " . $q->quote($orig_tbl->{db}, "${prefix}_$name") . ";"; @@ -11806,7 +11812,8 @@ you notice queueing, it is best to decrease the chunk time. =item --preserve-triggers -Preserves old triggers when specified. +Preserves old triggers when specified. This mode will briefly hold up to 3 +locks when it rewrites original triggers to include the tool code. =item --new-table-name From b4568ee2553edc1acd16f50bcbc3ffe2219028fa Mon Sep 17 00:00:00 2001 From: Manjot Singh Date: Fri, 27 May 2016 08:55:37 -0700 Subject: [PATCH 7/9] Fixed small issue with old trigger run when no old triggers line 9554 --- bin/pt-online-schema-change | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bin/pt-online-schema-change b/bin/pt-online-schema-change index d4ce86c3..58f6a77f 100755 --- a/bin/pt-online-schema-change +++ b/bin/pt-online-schema-change @@ -9551,9 +9551,11 @@ sub main { print "Adding original triggers to new table.\n"; foreach my $orig_trigger (@orig_triggers) { - PTDEBUG && _d($orig_trigger); - print $orig_trigger, "\n" if $o->get('print'); - $cxn->dbh()->do($orig_trigger); + if ($orig_trigger ne '') { + PTDEBUG && _d($orig_trigger); + print $orig_trigger, "\n" if $o->get('print'); + $cxn->dbh()->do($orig_trigger); + } } } From 5642cc4fd058772d5d9ce80e29c8e7439ce9f092 Mon Sep 17 00:00:00 2001 From: Manjot Singh Date: Fri, 27 May 2016 21:00:14 -0700 Subject: [PATCH 8/9] Bug fixes for pt-osc --preserve-triggers after sysbench tests --- bin/pt-online-schema-change | 43 ++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/bin/pt-online-schema-change b/bin/pt-online-schema-change index 58f6a77f..9438b758 100755 --- a/bin/pt-online-schema-change +++ b/bin/pt-online-schema-change @@ -10615,13 +10615,14 @@ sub create_triggers { . "WHERE $del_index_cols"; if($delete_trigger_ins_point) { + $delete_trigger = "LOCK TABLES $orig_tbl->{name} WRITE;\n" . "DROP TRIGGER IF EXISTS `$orig_tbl->{db}`.`$orig_delete_trigger->[1]`;\n" . $delete_trigger . "\n" - . substr($orig_delete_trigger->[3], 0, $delete_trigger_ins_point) + . terminate_sql(substr($orig_delete_trigger->[3], 0, $delete_trigger_ins_point))."\n" . $delete_trigger_action . ";\n" - . substr($orig_delete_trigger->[3], $delete_trigger_ins_point) . ";\n" + . "END;\n" . "UNLOCK TABLES;"; } else { @@ -10630,7 +10631,7 @@ sub create_triggers { . "DROP TRIGGER IF EXISTS `$orig_tbl->{db}`.`$orig_delete_trigger->[1]`;\n" . $delete_trigger . "\n" . "BEGIN\n" - . $orig_delete_trigger->[3] . "\n" + . terminate_sql($orig_delete_trigger->[3]) . "\n" . $delete_trigger_action . ";\n" . "END;\n" . "UNLOCK TABLES;"; @@ -10644,7 +10645,7 @@ sub create_triggers { = "CREATE DEFINER=$definer TRIGGER `$orig_delete_trigger->[0]`.`$orig_delete_trigger->[1]` " . "AFTER DELETE ON $new_tbl->{name}\n" . "FOR EACH ROW\n" - . $orig_delete_trigger->[3] . ";\n"; + . terminate_sql($orig_delete_trigger->[3]) . "\n"; } else { $delete_trigger @@ -10658,7 +10659,6 @@ sub create_triggers { # find if trigger has begin/end - supply insertion point my $insert_trigger_ins_point = trigger_ins_point(trigger => $orig_insert_trigger->[3]); - my $update_trigger_ins_point = trigger_ins_point(trigger => $orig_update_trigger->[3]); $insert_trigger = "CREATE TRIGGER `${prefix}_ins` AFTER INSERT ON $orig_tbl->{name} " @@ -10672,9 +10672,9 @@ sub create_triggers { = "LOCK TABLES $orig_tbl->{name} WRITE;\n" . "DROP TRIGGER IF EXISTS `$orig_tbl->{db}`.`$orig_insert_trigger->[1]`;\n" . $insert_trigger . "\n" - . substr($orig_insert_trigger->[3], 0, $insert_trigger_ins_point) + . terminate_sql(substr($orig_insert_trigger->[3], 0, $insert_trigger_ins_point)) . "\n" . $insert_trigger_action . ";\n" - . substr($orig_insert_trigger->[3], $insert_trigger_ins_point) . ";\n" + . "END;\n" . "UNLOCK TABLES;"; } else { @@ -10683,7 +10683,7 @@ sub create_triggers { . "DROP TRIGGER IF EXISTS `$orig_tbl->{db}`.`$orig_insert_trigger->[1]`;\n" . $insert_trigger . "\n" . "BEGIN\n" - . $orig_insert_trigger->[3] . "\n" + . terminate_sql($orig_insert_trigger->[3]) . "\n" . $insert_trigger_action . ";\n" . "END; \n" . "UNLOCK TABLES;"; @@ -10697,7 +10697,7 @@ sub create_triggers { = "CREATE DEFINER=$definer TRIGGER `$orig_insert_trigger->[0]`.`$orig_insert_trigger->[1]` " . "AFTER INSERT ON $new_tbl->{name}\n" . "FOR EACH ROW\n" - . $orig_insert_trigger->[3] . ";\n"; + . terminate_sql($orig_insert_trigger->[3]) . "\n"; } else { $insert_trigger @@ -10725,9 +10725,9 @@ sub create_triggers { = "LOCK TABLES $orig_tbl->{name} WRITE;\n" . "DROP TRIGGER IF EXISTS `$orig_tbl->{db}`.`$orig_update_trigger->[1]`;\n" . $update_trigger . "\n" - . substr($orig_update_trigger->[3], 0, $update_trigger_ins_point) + . terminate_sql(substr($orig_update_trigger->[3], 0, $update_trigger_ins_point)) . "\n" . $update_trigger_action . ";\n" - . substr($orig_update_trigger->[3], $update_trigger_ins_point) . ";\n" + . "END;\n" . "UNLOCK TABLES;"; } else { @@ -10736,7 +10736,7 @@ sub create_triggers { . "DROP TRIGGER IF EXISTS `$orig_tbl->{db}`.`$orig_update_trigger->[1]`;\n" . $update_trigger . "\n" . "BEGIN\n" - . $orig_update_trigger->[3] . "\n" + . terminate_sql($orig_update_trigger->[3]) . "\n" . $update_trigger_action . ";\n" . "END;\n" . "UNLOCK TABLES;"; @@ -10750,7 +10750,7 @@ sub create_triggers { = "CREATE DEFINER=$definer TRIGGER `$orig_update_trigger->[0]`.`$orig_update_trigger->[1]` " . "AFTER UPDATE ON $new_tbl->{name}\n" . "FOR EACH ROW\n" - . $orig_update_trigger->[3] . ";\n"; + . terminate_sql($orig_update_trigger->[3]) . "\n"; } else { @@ -11078,6 +11078,7 @@ sub ts { return $msg ? "$ts $msg" : $ts; } +# find point in trigger we can insert pt-osc code for --preserve-triggers sub trigger_ins_point { my ( %args ) = @_; my @required_args = qw(trigger); @@ -11095,6 +11096,22 @@ sub trigger_ins_point { return $ins_point; } +# sub to add ; if line doesn't end in ; +sub terminate_sql { + my ( $text ) = @_; + die "I need a text argument" unless defined $text; + $text = trim($text); + if(substr($text, -1) ne ';') { $text .= ';'; } + return $text; +} + +sub trim { + my ( $text ) = @_; + die "I need a text argument" unless defined $text; + $text =~ s/^\s+|\s+$//gm; + return $text; +} + # Catches signals so we can exit gracefully. sub sig_int { my ( $signal ) = @_; From a564cc7bc88ecd62010d4f8a3a5b27ebfa1ba427 Mon Sep 17 00:00:00 2001 From: Manjot Singh Date: Tue, 28 Jun 2016 12:33:44 -0700 Subject: [PATCH 9/9] Added changes suggested by mbenshoof, michaeldg - Fixed issue with dry run and preserve triggers - Fixed issue with trim multiline - Changed DSNParser in lib --- bin/pt-online-schema-change | 6 ++++-- lib/DSNParser.pm | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/pt-online-schema-change b/bin/pt-online-schema-change index 9438b758..177c241d 100755 --- a/bin/pt-online-schema-change +++ b/bin/pt-online-schema-change @@ -9554,7 +9554,9 @@ sub main { if ($orig_trigger ne '') { PTDEBUG && _d($orig_trigger); print $orig_trigger, "\n" if $o->get('print'); - $cxn->dbh()->do($orig_trigger); + if ( $o->get('execute') ) { + $cxn->dbh()->do($orig_trigger); + } } } } @@ -11108,7 +11110,7 @@ sub terminate_sql { sub trim { my ( $text ) = @_; die "I need a text argument" unless defined $text; - $text =~ s/^\s+|\s+$//gm; + $text =~ s/^\s+|\s+$//g; return $text; } diff --git a/lib/DSNParser.pm b/lib/DSNParser.pm index 9ec39bd3..61cc4366 100644 --- a/lib/DSNParser.pm +++ b/lib/DSNParser.pm @@ -276,6 +276,7 @@ sub get_dbh { PrintError => 0, ShowErrorStatement => 1, mysql_enable_utf8 => ($cxn_string =~ m/charset=utf8/i ? 1 : 0), + mysql_multi_statements => 1, }; @{$defaults}{ keys %$opts } = values %$opts; if (delete $defaults->{L}) { # L for LOAD DATA LOCAL INFILE, our own extension