Merge branch '3.0' into PT-139

bin/pt-archiver

@@ -45,7 +45,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.3';
+our $VERSION = '3.0.4';
 
 use strict;
 use warnings FATAL => 'all';
@@ -1954,7 +1954,7 @@ sub parse {
    my ($name)     = $ddl =~ m/CREATE (?:TEMPORARY )?TABLE\s+(`.+?`)/;
    (undef, $name) = $self->{Quoter}->split_unquote($name) if $name;
 
-   $ddl =~ s/(`[^`]+`)/\L$1/g;
+   $ddl =~ s/(`[^`\n]+`)/\L$1/gm;
 
    my $engine = $self->get_engine($ddl);
 
@@ -1965,8 +1965,8 @@ sub parse {
    my %def_for;
    @def_for{@cols} = @defs;
 
-   my (@nums, @null);
-   my (%type_for, %is_nullable, %is_numeric, %is_autoinc);
+   my (@nums, @null, @non_generated);
+   my (%type_for, %is_nullable, %is_numeric, %is_autoinc, %is_generated);
    foreach my $col ( @cols ) {
       my $def = $def_for{$col};
 
@@ -1983,6 +1983,11 @@ sub parse {
          push @null, $col;
          $is_nullable{$col} = 1;
       }
+      if ( remove_quoted_text($def) =~ m/\WGENERATED\W/i ) {
+          $is_generated{$col} = 1;
+      } else {
+          push @non_generated, $col;
+      }
       $is_autoinc{$col} = $def =~ m/AUTO_INCREMENT/i ? 1 : 0;
    }
 
@@ -1991,24 +1996,34 @@ sub parse {
    my ($charset) = $ddl =~ m/DEFAULT CHARSET=(\w+)/;
 
    return {
-      name           => $name,
-      cols           => \@cols,
-      col_posn       => { map { $cols[$_] => $_ } 0..$#cols },
-      is_col         => { map { $_ => 1 } @cols },
-      null_cols      => \@null,
-      is_nullable    => \%is_nullable,
-      is_autoinc     => \%is_autoinc,
-      clustered_key  => $clustered_key,
-      keys           => $keys,
-      defs           => \%def_for,
-      numeric_cols   => \@nums,
-      is_numeric     => \%is_numeric,
-      engine         => $engine,
-      type_for       => \%type_for,
-      charset        => $charset,
+      name               => $name,
+      cols               => \@cols,
+      col_posn           => { map { $cols[$_] => $_ } 0..$#cols },
+      is_col             => { map { $_ => 1 } @non_generated },
+      null_cols          => \@null,
+      is_nullable        => \%is_nullable,
+      non_generated_cols => \@non_generated,
+      is_autoinc         => \%is_autoinc,
+      is_generated       => \%is_generated,
+      clustered_key      => $clustered_key,
+      keys               => $keys,
+      defs               => \%def_for,
+      numeric_cols       => \@nums,
+      is_numeric         => \%is_numeric,
+      engine             => $engine,
+      type_for           => \%type_for,
+      charset            => $charset,
    };
 }
 
+sub remove_quoted_text {
+   my ($string) = @_;
+   $string =~ s/[^\\]`[^`]*[^\\]`//g; 
+   $string =~ s/[^\\]"[^"]*[^\\]"//g; 
+   $string =~ s/[^\\]"[^"]*[^\\]"//g; 
+   return $string;
+}
+
 sub sort_indexes {
    my ( $self, $tbl ) = @_;
 
@@ -2085,6 +2100,8 @@ sub check_table {
    my $db_tbl = $q->quote($db, $tbl);
    PTDEBUG && _d('Checking', $db_tbl);
 
+   $self->{check_table_error} = undef;
+
    my $sql = "SHOW TABLES FROM " . $q->quote($db)
            . ' LIKE ' . $q->literal_like($tbl);
    PTDEBUG && _d($sql);
@@ -2092,8 +2109,9 @@ sub check_table {
    eval {
       $row = $dbh->selectrow_arrayref($sql);
    };
-   if ( $EVAL_ERROR ) {
-      PTDEBUG && _d($EVAL_ERROR);
+   if ( my $e = $EVAL_ERROR ) {
+      PTDEBUG && _d($e);
+      $self->{check_table_error} = $e;
       return 0;
    }
    if ( !$row->[0] || $row->[0] ne $tbl ) {
@@ -4518,7 +4536,8 @@ sub lost_connection {
    my ($self, $e) = @_;
    return 0 unless $e;
    return $e =~ m/MySQL server has gone away/
-       || $e =~ m/Lost connection to MySQL server/;
+       || $e =~ m/Lost connection to MySQL server/
+       || $e =~ m/Server shutdown in progress/;
 }
 
 sub dbh {
@@ -4537,6 +4556,11 @@ sub name {
    return $self->{hostname} || $self->{dsn_name} || 'unknown host';
 }
 
+sub description {
+   my ($self) = @_;
+   return sprintf("%s -> %s:%s", $self->name(), $self->{dsn}->{h}, $self->{dsn}->{P} || 'socket');
+}
+
 sub get_id {
    my ($self, $cxn) = @_;
 
@@ -5913,6 +5937,8 @@ my $get_sth;
 my ( $OUT_OF_RETRIES, $ROLLED_BACK, $ALL_IS_WELL ) = ( 0, -1, 1 );
 my ( $src, $dst );
 my $pxc_version = '0';
+my $fields_separated_by = "\t";
+my $optionally_enclosed_by;
 
 # Holds the arguments for the $sth's bind variables, so it can be re-tried
 # easily.
@@ -6314,10 +6340,8 @@ sub main {
    # this ensures returned sets are disjoint when ran on partitioned tables
    # issue 1376561
    my $index_cols;
-   if (  $sel_stmt->{index} 
-         && $src->{info}->{keys}->{$sel_stmt->{index}}->{cols} 
-   ) {
-      $index_cols = $src->{info}->{keys}->{$sel_stmt->{index}}->{colnames};
+   if (  $sel_stmt->{index} && $src->{info}->{keys}->{$sel_stmt->{index}}->{cols} ) {
+      $index_cols = join(",",map { "`$_`" } @{$src->{info}->{keys}->{$sel_stmt->{index}}->{cols}});
    }
 
    foreach my $thing ( $first_sql, $next_sql ) {
@@ -6542,12 +6566,19 @@ sub main {
    
    # Open the file and print the header to it. 
    if ( $archive_file ) { 
+      if ($o->got('output-format') && $o->get('output-format') ne 'dump' && $o->get('output-format') ne 'csv') {
+          warn "Invalid output format:". $o->get('format');
+          warn "Using default 'dump' format";
+      } elsif ($o->get('output-format') || '' eq 'csv') {
+              $fields_separated_by = ", ";
+              $optionally_enclosed_by = '"';
+      }
       my $need_hdr = $o->get('header') && !-f $archive_file; 
       $archive_fh = IO::File->new($archive_file, ">>$charset") 
          or die "Cannot open $charset $archive_file: $OS_ERROR\n"; 
       $archive_fh->autoflush(1) unless $o->get('buffer'); 
       if ( $need_hdr ) { 
-         print { $archive_fh } '', escape(\@sel_cols), "\n" 
+         print { $archive_fh } '', escape(\@sel_cols, $fields_separated_by, $optionally_enclosed_by), "\n" 
             or die "Cannot write to $archive_file: $OS_ERROR\n"; 
       } 
    } 
@@ -6592,7 +6623,7 @@ sub main {
          # problem, hopefully the data has at least made it to the file.
          my $escaped_row;
          if ( $archive_fh || $bulkins_file ) {
-            $escaped_row = escape([@{$row}[@sel_slice]]);
+            $escaped_row = escape([@{$row}[@sel_slice]], $fields_separated_by, $optionally_enclosed_by);
          }
          if ( $archive_fh ) {
             trace('print_file', sub {
@@ -7049,11 +7080,18 @@ sub do_with_retries {
 # described in the LOAD DATA INFILE section of the MySQL manual,
 # http://dev.mysql.com/doc/refman/5.0/en/load-data.html
 sub escape {
-   my ($row) = @_;
-   return join("\t", map {
+   my ($row, $fields_separated_by, $optionally_enclosed_by) = @_;
+   $fields_separated_by ||= "\t";
+   $optionally_enclosed_by ||= '';
+ 
+   return join($fields_separated_by, map {
       s/([\t\n\\])/\\$1/g if defined $_;  # Escape tabs etc
-      defined $_ ? $_ : '\N';             # NULL = \N
+      $_ = defined $_ ? $_ : '\N';             # NULL = \N
+      # var & ~var will return 0 only for numbers
+      $_ =~ s/([^\\])"/$1\\"/g if ($_ !~ /^[0-9,.E]+$/  && $optionally_enclosed_by eq '"');
+      $_ = $optionally_enclosed_by && $_ & ~$_ ? $optionally_enclosed_by."$_".$optionally_enclosed_by : $_;
    } @$row);
+
 }
 
 sub ts {
@@ -7674,6 +7712,17 @@ Runs OPTIMIZE TABLE after finishing.  See L<"--analyze"> for the option syntax
 and L<http://dev.mysql.com/doc/en/optimize-table.html> for details on OPTIMIZE
 TABLE.
 
+=item --output-format
+
+type: string
+
+Used with L<"--file"> to specify the output format.
+
+Valid formats are:
+  dump: MySQL dump format using tabs as field separator (default)
+  csv : Dump rows using ',' as separator and optionally enclosing fields by '"'.
+        This format is equivalent to FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"'.
+
 =item --password
 
 short form: -p; type: string
@@ -8460,6 +8509,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-archiver 3.0.2
+pt-archiver 3.0.4
 
 =cut