update pod selection and confirming crashed state

Co-authored-by: Andrew Pogrebnoy <absourd.noise@gmail.com>

.travis.yml

@@ -36,6 +36,9 @@ env:
     - MINIO_ENDPOINT: http://localhost:9000/
     - MINIO_ACCESS_KEY_ID: example00000
     - MINIO_SECRET_ACCESS_KEY: secret00000
+    # REVIEWDOG_GITHUB_API_TOKEN
+    - secure: "px8XYeNEAFTSTb1hYZuEOxqOXUxvp3EoU+KCtPck/KNozkoS95eBd9klgr3Os4wPKloLdMhrr0VE98lukogUxA/NmnYnos01kegjWgwwM6fkob8JxaN5KK4oUFF1wmirBlrjGlw8vUErPwINmrK4BywKpDbw6Yip6FzxdlWESHI="
+
   matrix:
     - MONGODB_IMAGE=mongo:3.0
     - MONGODB_IMAGE=mongo:3.2
@@ -58,8 +61,14 @@ before_install:
 
 install:
   - go get -u github.com/golang/dep/cmd/dep
+  # install reviewdog and golangci-lin
+  - curl https://raw.githubusercontent.com/reviewdog/reviewdog/master/install.sh| sh -s
+  - curl https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s latest
 
 before_script:
+  # static analyze
+  - bin/golangci-lint run -c=.golangci-required.yml --out-format=line-number | bin/reviewdog -f=golangci-lint -level=error -reporter=github-pr-check
+  - bin/golangci-lint run -c=.golangci.yml --out-format=line-number | bin/reviewdog -f=golangci-lint -level=error -reporter=github-pr-review
   # log versions
   - docker --version
   - docker-compose --version
@@ -69,7 +78,7 @@ before_script:
   - dep ensure
 
 script:
-  - docker ps 
+  - docker ps
   - go test -timeout 20m ./src/go/...
 
 allow_failures:

Gopkg.lock

@@ -188,12 +188,12 @@
   revision = "197f4ad8db8d1b04ff408042119176907c971f0a"
 
 [[projects]]
-  digest = "1:1d7e1867c49a6dd9856598ef7c3123604ea3daabf5b83f303ff457bcbc410b1d"
+  digest = "1:c45802472e0c06928cd997661f2af610accd85217023b1d5f6331bebce0671d3"
   name = "github.com/pkg/errors"
   packages = ["."]
   pruneopts = ""
-  revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4"
-  version = "v0.8.1"
+  revision = "614d223910a179a466c1767a985424175c39b465"
+  version = "v0.9.1"
 
 [[projects]]
   digest = "1:55dcddb2ba6ab25098ee6b96f176f39305f1fde7ea3d138e7e10bb64a5bf45be"

bin/pt-archiver

@@ -7696,8 +7696,8 @@ Example:
 
 The file's contents are in the same format used by SELECT INTO OUTFILE, as
 documented in the MySQL manual: rows terminated by newlines, columns
-terminated by tabs, NULL characters are represented by \N, and special
-characters are escaped by \.  This lets you reload a file with LOAD DATA
+terminated by tabs, NULL characters are represented by C<\N>, and special
+characters are escaped by C<\>.  This lets you reload a file with LOAD DATA
 INFILE's default settings.
 
 If you want a column header at the top of the file, see L<"--header">.  The file
@@ -7856,8 +7856,10 @@ type: string
 Used with L<"--file"> to specify the output format.
 
 Valid formats are:
-  dump: MySQL dump format using tabs as field separator (default)
-  csv : Dump rows using ',' as separator and optionally enclosing fields by '"'.
+
+- dump: MySQL dump format using tabs as field separator (default)
+
+- csv : Dump rows using ',' as separator and optionally enclosing fields by '"'.
         This format is equivalent to FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"'.
 
 =item --password
@@ -7887,10 +7889,10 @@ Specify the Perl module name of a general-purpose plugin.  It is currently used
 only for statistics (see L<"--statistics">) and must have C<new()> and a
 C<statistics()> method.
 
-The C<new( src => $src, dst => $dst, opts => $o )> method gets the source
+The C<new( src =E<gt> $src, dst =E<gt> $dst, opts =E<gt> $o )> method gets the source
 and destination DSNs, and their database connections, just like the
 connection-specific plugins do.  It also gets an OptionParser object (C<$o>) for
-accessing command-line options (example: C<$o->get('purge');>).
+accessing command-line options (example: C<$o-E<gt>get('purge');>).
 
 The C<statistics(\%stats, $time)> method gets a hashref of the statistics
 collected by the archiving job, and the time the whole job started.
@@ -8230,7 +8232,7 @@ Percona Toolkit. Second, it checks for and warns about versions with known
 problems. For example, MySQL 5.5.25 had a critical bug and was re-released
 as 5.5.25a.
 
-A secure connection to Percona’s Version Check database server is done to
+A secure connection to Percona's Version Check database server is done to
 perform these checks. Each request is logged by the server, including software
 version numbers and unique ID of the checked system. The ID is generated by the
 Percona Toolkit installation script or when the Version Check database call is

bin/pt-config-diff

@@ -5747,7 +5747,7 @@ Percona Toolkit. Second, it checks for and warns about versions with known
 problems. For example, MySQL 5.5.25 had a critical bug and was re-released
 as 5.5.25a.
 
-A secure connection to Percona’s Version Check database server is done to
+A secure connection to Percona's Version Check database server is done to
 perform these checks. Each request is logged by the server, including software
 version numbers and unique ID of the checked system. The ID is generated by the
 Percona Toolkit installation script or when the Version Check database call is

bin/pt-deadlock-logger

@@ -5532,7 +5532,7 @@ Percona Toolkit. Second, it checks for and warns about versions with known
 problems. For example, MySQL 5.5.25 had a critical bug and was re-released
 as 5.5.25a.
 
-A secure connection to Percona’s Version Check database server is done to
+A secure connection to Percona's Version Check database server is done to
 perform these checks. Each request is logged by the server, including software
 version numbers and unique ID of the checked system. The ID is generated by the
 Percona Toolkit installation script or when the Version Check database call is

bin/pt-heartbeat

@@ -6386,19 +6386,6 @@ sub main {
          sleep $next_interval - $time;
          PTDEBUG && _d('Woke up at', ts(time));
 
-         if ( $o->get('check-read-only') && $o->get('update') ) {
-           my $read_only_interval = $o->get('read-only-interval') || $interval;
-           while (server_is_readonly($dbh)) {
-              PTDEBUG && _d("Server is read only. Sleeping for $read_only_interval seconds...");
-              sleep($read_only_interval);
-              if (
-                 -f $sentinel
-              ) {
-                 return 0;
-              }
-           }
-         }
-
          # Connect or reconnect if necessary.
          if ( !$dbh->ping() ) {
             $dbh = $dp->get_dbh($dp->get_cxn_params($dsn), { AutoCommit => 1 });
@@ -6409,6 +6396,17 @@ sub main {
             $heartbeat_sth = undef;
          }
 
+         if ( $o->get('check-read-only') && $o->get('update') ) {
+           my $read_only_interval = $o->get('read-only-interval') || $interval;
+           while (server_is_readonly($dbh)) {
+              PTDEBUG && _d("Server is read only. Sleeping for $read_only_interval seconds...");
+              sleep($read_only_interval);
+              if (-f $sentinel) {
+                  return 0;
+              }
+           }
+         }
+
          if ( $o->get('monitor') ) {
             $heartbeat_sth ||= $dbh->prepare($heartbeat_sql);
             my ($delay) = $get_delay->($heartbeat_sth);

bin/pt-kill

@@ -8323,9 +8323,9 @@ Print information to STDOUT about what is being done.
 
 These actions are taken for every matching query from all classes.
 The actions are taken in this order: L<"--print">, L<"--execute-command">,
-L<"--kill">/L<"--kill-query">.  This order allows L<"--execute-command">
+L<"--kill"> / L<"--kill-query">.  This order allows L<"--execute-command">
 to see the output of L<"--print"> and the query before
-L<"--kill">/L<"--kill-query">.  This may be helpful because pt-kill does
+L<"--kill"> / L<"--kill-query">.  This may be helpful because pt-kill does
 not pass any information to L<"--execute-command">.
 
 See also L<"GROUP, MATCH AND KILL">.

bin/pt-online-schema-change

@@ -8594,6 +8594,12 @@ sub main {
    # ########################################################################
    my $set_on_connect = sub {
       my ($dbh) = @_;
+      if (!$o->get('check-foreign-keys')) {
+            my $sql = "SET foreign_key_checks=0";
+            PTDEBUG && _d($sql);
+            print $sql, "\n" if $o->get('print');
+            $dbh->do($sql);
+      }
       return;
    };
 
@@ -9102,6 +9108,15 @@ sub main {
                $child_table->{name},
                $child_table->{row_est} || '?';
          }
+         
+         # TODO: Fix self referencing foreign keys handling.
+         # See: https://jira.percona.com/browse/PT-1802
+         #      https://jira.percona.com/browse/PT-1853
+         if (_has_self_ref_fks($orig_tbl->{db}, $orig_tbl->{tbl}, $child_tables) && $o->get('check-foreign-keys')) {
+             print "The table has self-referencing foreign keys and that might lead to errors.\n";
+             print "Use --no-check-foreign-keys to disable this check.\n";
+             return 1;
+         }
 
          if ( $alter_fk_method ) {
             # Let the user know how we're going to update the child table
@@ -10396,6 +10411,20 @@ sub check_alter {
    return;
 }
 
+sub _has_self_ref_fks { 
+    my ($orig_db, $orig_table, $child_tables) = @_;
+
+    my $db_tbl = sprintf('`%s`.`%s`', $orig_db, $orig_table);
+
+    foreach my $child_table ( @$child_tables ) {
+        if ("$db_tbl" eq "$child_table->{name}") {
+             return 1;
+         }
+    }
+
+    return 0;
+}
+
 # This function tries to detect if the --alter param is adding unique indexes.
 # It returns an array of arrays, having a list of fields for each unique index
 # found. 
@@ -11918,7 +11947,7 @@ The tool exits with an error if the host is a cluster node and the table
 is MyISAM or is being converted to MyISAM (C<ENGINE=MyISAM>), or if
 C<wsrep_OSU_method> is not C<TOI>.  There is no way to disable these checks.
 
-=head1 MySQL 5.7+ Generated columns
+=head1 MySQL 5.7 + Generated columns
 
 The tools ignores MySQL 5.7+ C<GENERATED> columns since the value for those columns
 is generated according to the expresion used to compute column values.
@@ -12123,7 +12152,7 @@ type: string
 Channel name used when connected to a server using replication channels.
 Suppose you have two masters, master_a at port 12345, master_b at port 1236 and
 a slave connected to both masters using channels chan_master_a and chan_master_b.
-If you want to run pt-table-sync to syncronize the slave against master_a, pt-table-sync
+If you want to run pt-table-sync to synchronize the slave against master_a, pt-table-sync
 won't be able to determine what's the correct master since SHOW SLAVE STATUS
 will return 2 rows. In this case, you can use --channel=chan_master_a to specify
 the channel name to use in the SHOW SLAVE STATUS command.
@@ -12168,6 +12197,15 @@ L<"--print"> and verify that the triggers are correct.
 
 =back
 
+=item --[no]check-foreign-keys
+
+default: yes
+
+Check for self-referencing foreign keys. Currently self referencing FKs are
+not full supported, so, to prevent errors, this program won't run if the table
+has self-referencing foreign keys. Use this parameter to disable self-referencing
+FK checks.
+
 =item --check-interval
 
 type: time; default: 1

bin/pt-slave-delay

@@ -4602,7 +4602,7 @@ server.  Before using this tool, please:
 C<pt-slave-delay> watches a slave and starts and stops its replication SQL
 thread as necessary to hold it at least as far behind the master as you
 request.  In practice, it will typically cause the slave to lag between
-L<"--delay"> and L<"--delay">+L<"--interval"> behind the master.
+L<"--delay"> and L<"--delay"> + L<"--interval"> behind the master.
 
 It bases the delay on binlog positions in the slave's relay logs by default,
 so there is no need to connect to the master.  This works well if the IO

bin/pt-stalk

@@ -1993,7 +1993,7 @@ then compared to L<"--threshold"> as usual.  The C<$EXT_ARGV> variable
 contains the MySQL options mentioned in the L<"SYNOPSIS"> above.
 
 The file should not alter the tool's existing global variables.  Prefix any
-file-specific global variables with "PLUGIN_" or make them local.
+file-specific global variables with C<PLUGIN_> or make them local.
 
 =item --help
 

bin/pt-table-checksum

@@ -13324,7 +13324,8 @@ first option on the command line.
 
 See the L<"--help"> output for a list of default config files.
 
-=item --[no]create-replicate-table
+=item --create-replicate-table
+=item --no-create-replicate-table
 
 default: yes
 
@@ -13687,7 +13688,7 @@ structure (MAGIC_create_replicate):
 
 Note: lower_boundary and upper_boundary data type can be BLOB. See L<"--binary-index">.
 
-By default, L<"--[no]create-replicate-table"> is true, so the database and
+By default, L<"--create-replicate-table"> is true, so the database and
 the table specified by this option are created automatically if they do not
 exist.
 

docs/index.rst

@@ -4,7 +4,7 @@
 ===============================
 
 Percona Toolkit is a collection of advanced command-line tools
-used by `Percona <http://www.percona.com/>`_) support staff
+used by `Percona <http://www.percona.com/>`_ support staff
 to perform a variety of MySQL, MongoDB, and system tasks
 that are too difficult or complex to perform manually.
 
@@ -59,6 +59,8 @@ Miscellaneous
    :maxdepth: 2
 
    bugs
+   ipv6_support
+   special_option_types
    authors
    copyright_license_and_warranty
    version

docs/release_notes.rst

@@ -1,7 +1,7 @@
 Percona Toolkit
 ***************
 
-v3.2.0 released 2019-04-23
+v3.2.0 released 2020-04-23
 ==========================
 
 Improvements:
@@ -27,6 +27,39 @@ Bug fixes:
 * :jirabug:`PT-1793`: ``pt-query-digest`` was unable to handle the year 2020 because of wrong ``tcpdump`` parsing. (Thank you, Kei Tsuchiya.)
 
 
+v3.1.0 released 2019-09-12
+==========================
+
+New Features:
+
+* :jirabug:`PT-1663`: Implement retention by bytes for pt-stalk
+
+Improvements:
+
+* :jirabug:`PT-1705`: Make pt-online-schema-change exit with different codes depending on the status
+* :jirabug:`PT-1761`: Prevent pt-osc to run under MySQL 8.0.14+ & 8.0.17
+* :jirabug:`PT-1746`: diskstats not working for kernel 4.18+
+
+Bugs Fixed:
+
+* :jirabug:`PT-1736`: pt-kill ignores --busy-time and --kill-busy-commands=Query when there is a process with Command=Execute
+* :jirabug:`PT-1575`: pt-mysql-summary does not print PXC section for PXC 5.6 and 5.7
+* :jirabug:`PT-1728`: Pt-table-checksum failing to scan small tables that get wiped out often
+* :jirabug:`PT-1720`: pt-pmp parses configuration files that lead to errors
+* :jirabug:`PT-1114`: LP #1182180: pt-table-checksum fails when table is empty
+* :jirabug:`PT-1715`: pt-upgrade documentation doesn't have the type tcpdump
+* :jirabug:`PT-1344`: LP #1580428: pt-online-schema-change: Use of uninitialized value $host in string
+* :jirabug:`PT-1492`: pt-kill in version 3.0.7 seems not to respect busy-time any longer
+* :jirabug:`PT-1798`: CLONE - yum repos do not contain 3.1.1 of percona toolkit
+* :jirabug:`PT-1797`: yum repos do not contain 3.1.1 of percona toolkit
+* :jirabug:`PT-1633`: pt-config-diff doesn't handle innodb_temp_data_file_path correctly
+* :jirabug:`PT-1630`: pt-table-checksum not working with galera cluster anymore since 3.0.11
+* :jirabug:`PT-1734`: Tailing log_error in pt-stalk doesn't work
+* :jirabug:`PT-1732`: Typo in link on percona.com
+
+
+
+  
 v3.0.13 released 2019-01-03
 ===========================
 
@@ -77,8 +110,7 @@ New features
 
 * :jirabug:`PT-1571`: Improved hostname recognition in ``pt-secure-collect``
 * :jirabug:`PT-1569`: Disabled ``--alter-foreign-keys-method=drop_swap`` in ``pt-online-schema-change``
-* :jirabug:`PT-242`: (``pt-stalk``) Include ``SHOW SLAVE STATUS`` on MySQL 5.7 (Thanks `Marcelo Altmann <https://www.p
-ercona.com/blog/author/marcelo-altmann/>`_)
+* :jirabug:`PT-242`: (``pt-stalk``) Include ``SHOW SLAVE STATUS`` on MySQL 5.7 (Thanks `Marcelo Altmann <https://www.percona.com/blog/author/marcelo-altmann/>`_)
 
 Fixed bugs
 
@@ -1105,17 +1137,17 @@ pt-query-digest --output json includes query examples as of v2.2.3. Some people
 When using drop swap with pt-online-schema-change there is some production impact. This impact can be measured because tool outputs the current timestamp on lines for operations that may take awhile.
 
 * Fixed bug #1163735: pt-table-checksum fails if explicit_defaults_for_timestamp is enabled in 5.6
-pt-table-checksum would fail if variable explicit_defaults_for_timestamp was enabled in MySQL 5.6.
+  pt-table-checksum would fail if variable explicit_defaults_for_timestamp was enabled in MySQL 5.6.
 
 * Fixed bug #1182856: Zero values causes "Invalid --set-vars value: var=0"
-Trying to assign 0 to any variable by using --set-vars option would cause “Invalid --set-vars value” message. 
+  Trying to assign 0 to any variable by using --set-vars option would cause “Invalid --set-vars value” message. 
 
 * Fixed bug #1188264: pt-online-schema-change error copying rows: Undefined subroutine &pt_online_schema_change::get
 
 * Fixed the typo in the pt-online-schema-change code that could lead to a tool crash when copying the rows.
 
 * Fixed bug #1199591: pt-table-checksum doesn't use non-unique index with highest cardinality
-pt-table-checksum was using the first non-unique index instead of the one with the highest cardinality due to a sorting bug.
+  pt-table-checksum was using the first non-unique index instead of the one with the highest cardinality due to a sorting bug.
 
 Percona Toolkit packages can be downloaded from
 http://www.percona.com/downloads/percona-toolkit/ or the Percona Software

docs/rn.3-1-0.txt

@@ -0,0 +1,44 @@
+.. _PT-3.1.0:
+
+================================================================================
+*Percona Toolkit* 3.1.0
+================================================================================
+
+:Date: September 12, 2019
+:Installation: `Installing Percona Toolkit <https://www.percona.com/doc/percona-toolkit/LATEST/installation.html>`_
+
+New Features
+================================================================================
+
+* :jirabug:`PT-1663`: Implement retention by bytes for pt-stalk
+
+
+
+Improvements
+================================================================================
+
+* :jirabug:`PT-1705`: Make pt-online-schema-change exit with different codes depending on the status
+* :jirabug:`PT-1761`: Prevent pt-osc to run under MySQL 8.0.14+ & 8.0.17
+* :jirabug:`PT-1746`: diskstats not working for kernel 4.18+
+
+
+
+Bugs Fixed
+================================================================================
+
+* :jirabug:`PT-1736`: pt-kill ignores --busy-time and --kill-busy-commands=Query when there is a process with Command=Execute
+* :jirabug:`PT-1575`: pt-mysql-summary does not print PXC section for PXC 5.6 and 5.7
+* :jirabug:`PT-1728`: Pt-table-checksum failing to scan small tables that get wiped out often
+* :jirabug:`PT-1720`: pt-pmp parses configuration files that lead to errors
+* :jirabug:`PT-1114`: LP #1182180: pt-table-checksum fails when table is empty
+* :jirabug:`PT-1715`: pt-upgrade documentation doesn't have the type tcpdump
+* :jirabug:`PT-1344`: LP #1580428: pt-online-schema-change: Use of uninitialized value $host in string
+* :jirabug:`PT-1492`: pt-kill in version 3.0.7 seems not to respect busy-time any longer
+* :jirabug:`PT-1798`: CLONE - yum repos do not contain 3.1.1 of percona toolkit
+* :jirabug:`PT-1797`: yum repos do not contain 3.1.1 of percona toolkit
+* :jirabug:`PT-1633`: pt-config-diff doesn't handle innodb_temp_data_file_path correctly
+* :jirabug:`PT-1630`: pt-table-checksum not working with galera cluster anymore since 3.0.11
+* :jirabug:`PT-1734`: Tailing log_error in pt-stalk doesn't work
+* :jirabug:`PT-1732`: Typo in link on percona.com
+
+

src/go/Makefile

@@ -18,7 +18,7 @@ BIN_DIR=$(shell git rev-parse --show-toplevel)/bin
 SRC_DIR=$(shell git rev-parse --show-toplevel)/src/go
 LDFLAGS="-X main.Version=${VERSION} -X main.Build=${BUILD} -X main.GoVersion=${GOVERSION} -X main.Commit=${COMMIT} -s -w"
 
-TEST_PSMDB_VERSION?=3.6
+TEST_PSMDB_VERSION?=4.0
 TEST_MONGODB_FLAVOR?=percona/percona-server-mongodb
 TEST_MONGODB_ADMIN_USERNAME?=admin
 TEST_MONGODB_ADMIN_PASSWORD?=admin123456

src/go/docker-compose.yml

@@ -4,13 +4,10 @@ services:
   standalone:
     network_mode: host
     image: ${TEST_MONGODB_FLAVOR}:${TEST_PSMDB_VERSION}
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: ${TEST_MONGODB_ADMIN_USERNAME}
+      MONGO_INITDB_ROOT_PASSWORD: ${TEST_MONGODB_ADMIN_PASSWORD}
     command: --port=27017 
-    volumes:
-    - ./docker/test/entrypoint-mongod.sh:/entrypoint.sh:ro
-    - ./docker/test/entrypoint-mongod.sh:/usr/local/bin/docker-entrypoint.sh:ro
-    - ./docker/test/mongod.key:/mongod.key:ro
-    - ./docker/test/ssl/rootCA.crt:/rootCA.crt:ro
-    - ./docker/test/ssl/mongodb.pem:/mongod.pem:ro
   s1-mongo1:
     network_mode: host
     image: ${TEST_MONGODB_FLAVOR}:${TEST_PSMDB_VERSION}

src/go/internal/testutils/env.go

@@ -30,6 +30,8 @@ const (
 	envMongoDBConfigsvr3Port       = "TEST_MONGODB_CONFIGSVR3_PORT"
 	//
 	envMongoDBMongosPort = "TEST_MONGODB_MONGOS_PORT"
+
+	envMongoDBStandalonePort = "TEST_MONGODB_STANDALONE_PORT"
 	//
 	envMongoDBUser     = "TEST_MONGODB_ADMIN_USERNAME"
 	envMongoDBPassword = "TEST_MONGODB_ADMIN_PASSWORD"
@@ -39,6 +41,9 @@ var (
 	// MongoDBHost is the hostname. Since it runs locally, it is localhost
 	MongoDBHost = "127.0.0.1"
 
+	// Port for standalone instance
+	MongoDBStandalonePort = os.Getenv(envMongoDBStandalonePort)
+
 	// MongoDBShard1ReplsetName Replicaset name for shard 1
 	MongoDBShard1ReplsetName = os.Getenv(envMongoDBShard1ReplsetName)
 	// MongoDBShard1PrimaryPort is the port for the primary instance of shard 1

src/go/mongolib/util/util.go

@@ -2,7 +2,6 @@ package util
 
 import (
 	"context"
-	"fmt"
 	"sort"
 	"strings"
 
@@ -13,8 +12,13 @@ import (
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
 
+const (
+	shardingNotEnabledErrorCode = 203
+)
+
 var (
-	CANNOT_GET_QUERY_ERROR = errors.New("cannot get query field from the profile document (it is not a map)")
+	CannotGetQueryError     = errors.New("cannot get query field from the profile document (it is not a map)")
+	ShardingNotEnabledError = errors.New("sharding not enabled")
 )
 
 func GetReplicasetMembers(ctx context.Context, clientOptions *options.ClientOptions) ([]proto.Members, error) {
@@ -92,7 +96,7 @@ func GetReplicasetMembers(ctx context.Context, clientOptions *options.ClientOpti
 			membersMap[m.Name] = m
 		}
 
-		client.Disconnect(ctx)
+		client.Disconnect(ctx) //nolint
 	}
 
 	for _, member := range membersMap {
@@ -119,6 +123,9 @@ func GetHostnames(ctx context.Context, client *mongo.Client) ([]string, error) {
 	var shardsMap proto.ShardsMap
 	smRes := client.Database("admin").RunCommand(ctx, primitive.M{"getShardMap": 1})
 	if smRes.Err() != nil {
+		if e, ok := smRes.Err().(mongo.CommandError); ok && e.Code == shardingNotEnabledErrorCode {
+			return nil, ShardingNotEnabledError // standalone instance
+		}
 		return nil, errors.Wrap(smRes.Err(), "cannot getShardMap for GetHostnames")
 	}
 	if err := smRes.Decode(&shardsMap); err != nil {
@@ -134,7 +141,8 @@ func GetHostnames(ctx context.Context, client *mongo.Client) ([]string, error) {
 		}
 	}
 
-	return nil, fmt.Errorf("cannot get shards map")
+	// Some MongoDB servers won't return ShardingNotEnabledError for stand alone instances.
+	return nil, nil // standalone instance
 }
 
 func buildHostsListFromReplStatus(replStatus proto.ReplicaSetStatus) []string {
@@ -265,7 +273,7 @@ func GetQueryField(doc proto.SystemProfile) (primitive.M, error) {
 				if ssquery, ok := squery.(primitive.M); ok {
 					return ssquery, nil
 				}
-				return nil, CANNOT_GET_QUERY_ERROR
+				return nil, CannotGetQueryError
 			}
 		}
 	}
@@ -301,7 +309,7 @@ func GetQueryField(doc proto.SystemProfile) (primitive.M, error) {
 		if ssquery, ok := squery.(primitive.M); ok {
 			return ssquery, nil
 		}
-		return nil, CANNOT_GET_QUERY_ERROR
+		return nil, CannotGetQueryError
 	}
 
 	// "query" in MongoDB 3.2+ is better structured and always has a "filter" subkey:
@@ -309,7 +317,7 @@ func GetQueryField(doc proto.SystemProfile) (primitive.M, error) {
 		if ssquery, ok := squery.(primitive.M); ok {
 			return ssquery, nil
 		}
-		return nil, CANNOT_GET_QUERY_ERROR
+		return nil, CannotGetQueryError
 	}
 
 	// {"ns":"test.system.js","op":"query","query":{"find":"system.js"}}

src/go/mongolib/util/util_test.go

@@ -14,24 +14,34 @@ import (
 
 func TestGetHostnames(t *testing.T) {
 	testCases := []struct {
-		name string
-		uri  string
-		want []string
+		name      string
+		uri       string
+		want      []string
+		wantError bool
 	}{
 		{
-			name: "from_mongos",
-			uri:  fmt.Sprintf("mongodb://%s:%s@%s:%s", tu.MongoDBUser, tu.MongoDBPassword, tu.MongoDBHost, tu.MongoDBMongosPort),
-			want: []string{"127.0.0.1:17001", "127.0.0.1:17002", "127.0.0.1:17004", "127.0.0.1:17005", "127.0.0.1:17007"},
+			name:      "from_mongos",
+			uri:       fmt.Sprintf("mongodb://%s:%s@%s:%s", tu.MongoDBUser, tu.MongoDBPassword, tu.MongoDBHost, tu.MongoDBMongosPort),
+			want:      []string{"127.0.0.1:17001", "127.0.0.1:17002", "127.0.0.1:17004", "127.0.0.1:17005", "127.0.0.1:17007"},
+			wantError: false,
 		},
 		{
-			name: "from_mongod",
-			uri:  fmt.Sprintf("mongodb://%s:%s@%s:%s", tu.MongoDBUser, tu.MongoDBPassword, tu.MongoDBHost, tu.MongoDBShard1PrimaryPort),
-			want: []string{"127.0.0.1:17001", "127.0.0.1:17002", "127.0.0.1:17003"},
+			name:      "from_mongod",
+			uri:       fmt.Sprintf("mongodb://%s:%s@%s:%s", tu.MongoDBUser, tu.MongoDBPassword, tu.MongoDBHost, tu.MongoDBShard1PrimaryPort),
+			want:      []string{"127.0.0.1:17001", "127.0.0.1:17002", "127.0.0.1:17003"},
+			wantError: false,
 		},
 		{
-			name: "from_non_sharded",
-			uri:  fmt.Sprintf("mongodb://%s:%s@%s:%s", tu.MongoDBUser, tu.MongoDBPassword, tu.MongoDBHost, tu.MongoDBShard3PrimaryPort),
-			want: []string{"127.0.0.1:17021", "127.0.0.1:17022", "127.0.0.1:17023"},
+			name:      "from_non_sharded",
+			uri:       fmt.Sprintf("mongodb://%s:%s@%s:%s", tu.MongoDBUser, tu.MongoDBPassword, tu.MongoDBHost, tu.MongoDBShard3PrimaryPort),
+			want:      []string{"127.0.0.1:17021", "127.0.0.1:17022", "127.0.0.1:17023"},
+			wantError: false,
+		},
+		{
+			name:      "from_standalone",
+			uri:       fmt.Sprintf("mongodb://%s:%s@%s:%s", tu.MongoDBUser, tu.MongoDBPassword, tu.MongoDBHost, tu.MongoDBStandalonePort),
+			want:      nil,
+			wantError: true,
 		},
 	}
 
@@ -49,12 +59,12 @@ func TestGetHostnames(t *testing.T) {
 			}
 
 			hostnames, err := GetHostnames(ctx, client)
-			if err != nil {
-				t.Errorf("getHostnames: %v", err)
+			if err != nil && !test.wantError {
+				t.Errorf("Expecting error=nil, got: %v", err)
 			}
 
 			if !reflect.DeepEqual(hostnames, test.want) {
-				t.Errorf("Invalid hostnames from mongos. Got: %+v, want %+v", hostnames, test.want)
+				t.Errorf("Invalid hostnames. Got: %+v, want %+v", hostnames, test.want)
 			}
 		})
 	}
@@ -81,24 +91,34 @@ func TestGetServerStatus(t *testing.T) {
 
 func TestGetReplicasetMembers(t *testing.T) {
 	testCases := []struct {
-		name string
-		uri  string
-		want int
+		name    string
+		uri     string
+		want    int
+		wantErr bool
 	}{
 		{
-			name: "from_mongos",
-			uri:  fmt.Sprintf("mongodb://%s:%s@%s:%s", tu.MongoDBUser, tu.MongoDBPassword, tu.MongoDBHost, tu.MongoDBMongosPort),
-			want: 7,
+			name:    "from_mongos",
+			uri:     fmt.Sprintf("mongodb://%s:%s@%s:%s", tu.MongoDBUser, tu.MongoDBPassword, tu.MongoDBHost, tu.MongoDBMongosPort),
+			want:    7,
+			wantErr: false,
 		},
 		{
-			name: "from_mongod",
-			uri:  fmt.Sprintf("mongodb://%s:%s@%s:%s", tu.MongoDBUser, tu.MongoDBPassword, tu.MongoDBHost, tu.MongoDBShard1PrimaryPort),
-			want: 3,
+			name:    "from_mongod",
+			uri:     fmt.Sprintf("mongodb://%s:%s@%s:%s", tu.MongoDBUser, tu.MongoDBPassword, tu.MongoDBHost, tu.MongoDBShard1PrimaryPort),
+			want:    3,
+			wantErr: false,
 		},
 		{
-			name: "from_non_sharded",
-			uri:  fmt.Sprintf("mongodb://%s:%s@%s:%s", tu.MongoDBUser, tu.MongoDBPassword, tu.MongoDBHost, tu.MongoDBShard3PrimaryPort),
-			want: 3,
+			name:    "from_non_sharded",
+			uri:     fmt.Sprintf("mongodb://%s:%s@%s:%s", tu.MongoDBUser, tu.MongoDBPassword, tu.MongoDBHost, tu.MongoDBShard3PrimaryPort),
+			want:    3,
+			wantErr: false,
+		},
+		{
+			name:    "from_standalone",
+			uri:     fmt.Sprintf("mongodb://%s:%s@%s:%s", tu.MongoDBUser, tu.MongoDBPassword, tu.MongoDBHost, tu.MongoDBStandalonePort),
+			want:    0,
+			wantErr: true,
 		},
 	}
 
@@ -109,7 +129,7 @@ func TestGetReplicasetMembers(t *testing.T) {
 			defer cancel()
 
 			rsm, err := GetReplicasetMembers(ctx, clientOptions)
-			if err != nil {
+			if err != nil && !test.wantErr {
 				t.Errorf("Got an error while getting replicaset members: %s", err)
 			}
 			if len(rsm) != test.want {
@@ -146,7 +166,7 @@ func TestGetShardedHosts(t *testing.T) {
 		},
 	}
 
-	for _, test := range testCases {
+	for i, test := range testCases {
 		t.Run(test.name, func(t *testing.T) {
 			clientOptions := options.Client().ApplyURI(test.uri)
 			ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
@@ -156,6 +176,10 @@ func TestGetShardedHosts(t *testing.T) {
 			if err != nil {
 				t.Errorf("Cannot get a new client for host %s: %s", test.uri, err)
 			}
+			if client == nil {
+				t.Fatalf("mongodb client is nil i: %d, uri: %s\n", i, test.uri)
+			}
+
 			if err := client.Connect(ctx); err != nil {
 				t.Errorf("Cannot connect to host %s: %s", test.uri, err)
 			}

src/go/pt-k8s-pxc-recovery/kubectl/kubectl.go

@@ -0,0 +1,27 @@
+package kubectl
+
+import (
+	"errors"
+	"os/exec"
+	"runtime"
+)
+
+func getKubectl() string {
+	switch runtime.GOOS {
+	case "windows":
+		return "kubectl.exe"
+	default:
+		return "kubectl"
+	}
+}
+
+func RunCmd(namespace string, args ...string) (string, error) {
+	args = append([]string{"-v=0", "--namespace", namespace}, args...)
+	cmd := exec.Command(getKubectl(), args...)
+	stdouterr, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", errors.New(string(stdouterr))
+	}
+	output := string(stdouterr)
+	return output, nil
+}

src/go/pt-k8s-pxc-recovery/main.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/percona/percona-toolkit/src/go/pt-k8s-pxc-recovery/recover"
+)
+
+func stepOrError(err error) {
+	if err != nil {
+		log.Fatal("Error:", err)
+	}
+}
+
+func main() {
+	namespace, clusterName, debugImage := "", "", ""
+	flag.StringVar(&namespace, "namespace", "default", "Select the namespace in which the cluster is deployed in")
+	flag.StringVar(&clusterName, "cluster", "test-cluster", "Select the cluster to recover")
+	flag.StringVar(&debugImage, "debug-image", "percona/percona-xtradb-cluster:8.0.19-10.1-debug", "Name and version of the debug image to use")
+	flag.Parse()
+	c := recover.Cluster{Namespace: namespace, Name: clusterName}
+	log.SetPrefix("\n" + log.Prefix())
+
+	log.Printf("Starting recovery process")
+	go func() {
+		for {
+			time.Sleep(300 * time.Millisecond)
+			fmt.Print(".")
+		}
+	}()
+
+	log.Printf("Getting cluster size")
+	stepOrError(c.SetClusterSize())
+
+	log.Printf("Getting cluster image")
+	clusterImage, err := c.GetClusterImage()
+	stepOrError(err)
+
+	log.Printf("Confirming crashed status")
+	stepOrError(c.ConfirmCrashedStatus())
+
+	log.Printf("Patching cluster image")
+	stepOrError(c.PatchClusterImage(debugImage))
+
+	log.Printf("Restarting pods")
+	stepOrError(c.RestartPods())
+
+	log.Printf("Make sure pod zero is ready")
+	stepOrError(c.PodZeroReady())
+
+	log.Printf("Make sure all pods are running")
+	stepOrError(c.AllPodsRunning())
+
+	log.Print("Set SST in progress")
+	stepOrError(c.SetSSTInProgress())
+
+	log.Print("Waiting for all pods to be ready")
+	stepOrError(c.AllPodsReady())
+
+	log.Printf("Finding the most recent pod")
+	stepOrError(c.FindMostRecentPod())
+
+	log.Printf("Recovering most recent pod")
+	go func() {
+		err := c.RecoverMostRecentPod()
+		if err != nil {
+			log.Printf("Recovering most recent pod still in progress")
+		}
+	}()
+
+	time.Sleep(10 * time.Second)
+
+	log.Printf("Patching cluster image")
+	stepOrError(c.PatchClusterImage(clusterImage))
+
+	log.Printf("Restart all pods execpt most recent pod")
+	stepOrError(c.RestartAllPodsExceptMostRecent())
+
+	log.Printf("Make sure all pods are running")
+	stepOrError(c.AllPodsRunning())
+
+	log.Printf("Restart Most Recent Pod")
+	stepOrError(c.RestartMostRecentPod())
+
+	log.Print("Waiting for all pods to be ready")
+	stepOrError(c.AllPodsReady())
+
+	log.Printf("Completed the restore process")
+}

src/go/pt-k8s-pxc-recovery/recover/recover.go

@@ -0,0 +1,326 @@
+package recover
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/percona/percona-toolkit/src/go/pt-k8s-pxc-recovery/kubectl"
+)
+
+type Cluster struct {
+	Name          string
+	Size          int
+	MostRecentPod string
+	Namespace     string
+}
+
+func (c *Cluster) SetClusterSize() error {
+	args := []string{
+		"get",
+		"pxc",
+		c.Name,
+		"-o",
+		"jsonpath='{.spec.pxc.size}'",
+	}
+	strSize, err := kubectl.RunCmd(c.Namespace, args...)
+	if err != nil {
+		return err
+	}
+	strSize = strings.Trim(strSize, "'")
+	c.Size, err = strconv.Atoi(strSize)
+	if err != nil {
+		return fmt.Errorf("error getting cluster size, %s", err)
+	}
+	return nil
+}
+
+func (c *Cluster) GetClusterImage() (string, error) {
+	args := []string{
+		"get",
+		"pod",
+		c.Name + "-pxc-0",
+		"-o",
+		"jsonpath='{.spec.containers[0].image}'",
+	}
+	clusterImage, err := kubectl.RunCmd(c.Namespace, args...)
+	if err != nil {
+		return "", fmt.Errorf("Error getting cluster image %s", err)
+	}
+	clusterImage = strings.Trim(clusterImage, "'")
+	return clusterImage, nil
+}
+
+func (c *Cluster) getPods() ([]string, error) {
+	args := []string{
+		"get",
+		"pods",
+		"--no-headers",
+		"-o",
+		"custom-columns=:metadata.name",
+	}
+	out, err := kubectl.RunCmd(c.Namespace, args...)
+	if err != nil {
+		return []string{}, err
+	}
+	formatedOutput := strings.Split(out, "\n")
+	podNames := []string{}
+	for _, podName := range formatedOutput {
+		if strings.Contains(podName, c.Name) && strings.Contains(podName, "pxc") {
+			podNames = append(podNames, podName)
+		}
+	}
+	return podNames, nil
+}
+
+func (c *Cluster) ConfirmCrashedStatus() error {
+	podNames, err := c.getPods()
+	if err != nil {
+		return fmt.Errorf("Error getting pods : %s", err)
+	}
+	for _, pod := range podNames {
+		logs, err := kubectl.RunCmd(c.Namespace, "logs", pod)
+		if err != nil {
+			return fmt.Errorf("error confirming crashed cluster status %s", err)
+		}
+		if !strings.Contains(logs, "grastate.dat") && !strings.Contains(logs, "safe_to_bootstrap") &&
+			!strings.Contains(logs, "It may not be safe to bootstrap the cluster from this node") {
+			return fmt.Errorf("found one or more pods in healthy state, can't use recovery tool, please restart failed pods manually")
+		}
+	}
+	return nil
+}
+
+func (c *Cluster) PatchClusterImage(image string) error {
+	args := []string{
+		"patch",
+		"pxc",
+		c.Name,
+		"--type=merge",
+		`--patch={"spec":{"pxc":{"image":"` + image + `"}}}`,
+	}
+	_, err := kubectl.RunCmd(c.Namespace, args...)
+	return fmt.Errorf("error patching cluster image: %s", err)
+}
+
+func (c *Cluster) RestartPods() error {
+	podNames, err := c.getPods()
+	if err != nil {
+		return fmt.Errorf("error getting pods to restart pods: %s", err)
+	}
+	for _, podName := range podNames {
+		args := []string{
+			"delete",
+			"pod",
+			podName,
+			"--force",
+			"--grace-period=0",
+		}
+		_, err := kubectl.RunCmd(c.Namespace, args...)
+		if err != nil && !strings.Contains(err.Error(), "pods") && !strings.Contains(err.Error(), "not found") {
+			return fmt.Errorf("error restarting pods: %s", err)
+		}
+	}
+	return nil
+}
+
+func (c *Cluster) CheckPodReady(podName string) (bool, error) {
+	args := []string{
+		"get",
+		"pod",
+		podName,
+		"-o",
+		"jsonpath='{.status.containerStatuses[0].ready}'",
+	}
+	output, err := kubectl.RunCmd(c.Namespace, args...)
+	if err != nil {
+		return false, fmt.Errorf("error checking pod ready: %s", err)
+	}
+	return strings.Trim(output, "'") == "true", nil
+}
+
+func (c *Cluster) PodZeroReady() error {
+	podNames, err := c.getPods()
+	if err != nil {
+		return err
+	}
+	podZeroStatus := false
+	for !podZeroStatus {
+		time.Sleep(time.Second * 10)
+		podZeroStatus, err = c.CheckPodReady(podNames[0])
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *Cluster) CheckPodPhase(podName string, phase string) (bool, error) {
+	args := []string{
+		"get",
+		"pod",
+		podName,
+		"-o",
+		"jsonpath='{.status.phase}'",
+	}
+	output, err := kubectl.RunCmd(c.Namespace, args...)
+	if err != nil {
+		return false, fmt.Errorf("error checking pod phase: %s", err)
+	}
+	return strings.Trim(output, "'") == phase, nil
+}
+
+func (c *Cluster) AllPodsRunning() error {
+	podNames, err := c.getPods()
+	if err != nil {
+		return err
+	}
+	for _, podName := range podNames {
+		running := false
+		var err error
+		for !running {
+			time.Sleep(time.Second * 10)
+			running, err = c.CheckPodPhase(podName, "Running")
+			if err != nil && !strings.Contains(err.Error(), "NotFound") {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (c *Cluster) RunCommandInPod(podName string, cmd ...string) (string, error) {
+	args := []string{
+		"exec",
+		podName,
+		"--",
+	}
+	args = append(args, cmd...)
+	output, err := kubectl.RunCmd(c.Namespace, args...)
+	if err != nil {
+		return "", err
+	}
+	return output, nil
+}
+
+func (c *Cluster) SetSSTInProgress() error {
+	podNames, err := c.getPods()
+	if err != nil {
+		return err
+	}
+	for _, podName := range podNames {
+		_, err := c.RunCommandInPod(podName, "touch", "/var/lib/mysql/sst_in_progress")
+		if err != nil {
+			return fmt.Errorf("error setting sst in progress", err)
+		}
+	}
+	return nil
+}
+
+func (c *Cluster) AllPodsReady() error {
+	podNames, err := c.getPods()
+	if err != nil {
+		return err
+	}
+	for _, podName := range podNames {
+		podReadyStatus := false
+		for !podReadyStatus {
+			time.Sleep(time.Second * 10)
+			podReadyStatus, err = c.CheckPodReady(podName)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (c *Cluster) FindMostRecentPod() error {
+	podNames, err := c.getPods()
+	if err != nil {
+		return err
+	}
+	var recentPodName string
+	seqNo := 0
+	re := regexp.MustCompile(`(?m)seqno:\s*(\d*)`)
+	for _, podName := range podNames {
+		output, err := c.RunCommandInPod(podName, "cat", "/var/lib/mysql/grastate.dat")
+		if err != nil {
+			return err
+		}
+		match := re.FindStringSubmatch(output)
+		if len(match) < 2 {
+			return fmt.Errorf("error finding the most recent pod : unable to get seqno")
+		}
+		currentSeqNo, err := strconv.Atoi(string(match[1]))
+		if err != nil {
+			return err
+		}
+		if currentSeqNo > seqNo {
+			seqNo = currentSeqNo
+			recentPodName = podName
+		}
+	}
+	c.MostRecentPod = recentPodName
+	return nil
+}
+
+func (c *Cluster) RecoverMostRecentPod() error {
+	_, err := c.RunCommandInPod(c.MostRecentPod, "mysqld", "--wsrep_recover")
+	if err != nil {
+		return fmt.Errorf("error recovering most recent pod: %s", err)
+	}
+	_, err = c.RunCommandInPod(c.MostRecentPod, "bash", "-c", "sed -i 's/safe_to_bootstrap: 0/safe_to_bootstrap: 1/g' /var/lib/mysql/grastate.dat")
+	if err != nil {
+		return fmt.Errorf("error recovering most recent pod: %s", err)
+	}
+	_, err = c.RunCommandInPod(c.MostRecentPod, "bash", "-c", "sed -i 's/wsrep_cluster_address=.*/wsrep_cluster_address=gcomm:\\/\\//g' /etc/mysql/node.cnf")
+	if err != nil {
+		return fmt.Errorf("error recovering most recent pod: %s", err)
+	}
+	_, err = c.RunCommandInPod(c.MostRecentPod, "mysqld")
+	if err != nil {
+		return fmt.Errorf("error recovering most recent pod: %s", err)
+	}
+	return nil
+}
+
+func (c *Cluster) RestartAllPodsExceptMostRecent() error {
+	podNames, err := c.getPods()
+	if err != nil {
+		return err
+	}
+	for _, podName := range podNames {
+		if podName != c.MostRecentPod {
+			args := []string{
+				"delete",
+				"pod",
+				podName,
+				"--force",
+				"--grace-period=0",
+			}
+			_, err := kubectl.RunCmd(c.Namespace, args...)
+			if err != nil {
+				return fmt.Errorf("error restarting pods : %s", err)
+			}
+		}
+	}
+	return nil
+}
+
+func (c *Cluster) RestartMostRecentPod() error {
+	args := []string{
+		"delete",
+		"pod",
+		c.MostRecentPod,
+		"--force",
+		"--grace-period=0",
+	}
+	_, err := kubectl.RunCmd(c.Namespace, args...)
+	if err != nil {
+		return fmt.Errorf("error restarting most recent pod : %s", err)
+	}
+	return nil
+}

src/go/pt-mongodb-summary/main.go

@@ -38,13 +38,19 @@ const (
 	DefaultRunningOpsSamples  = 5
 	DefaultOutputFormat       = "text"
 	typeMongos                = "mongos"
+
+	// Exit Codes
+	cannotFormatResults              = 1
+	cannotParseCommandLineParameters = 2
+	cannotGetHostInfo                = 3
+	cannotGetReplicasetMembers       = 4
 )
 
 var (
 	Build     string = "2020-04-23" // nolint
 	GoVersion string = "1.14.1"     // nolint
-	Version   string = "3.2.0"      // nolint
-	Commit    string                // nolint
+	Version   string = "3.2.0"
+	Commit    string
 )
 
 type TimedStats struct {
@@ -158,7 +164,7 @@ func main() {
 	opts, err := parseFlags()
 	if err != nil {
 		log.Errorf("cannot get parameters: %s", err.Error())
-		os.Exit(2)
+		os.Exit(cannotParseCommandLineParameters)
 	}
 	if opts == nil && err == nil {
 		return
@@ -206,7 +212,7 @@ func main() {
 	defer client.Disconnect(ctx) // nolint
 
 	hostnames, err := util.GetHostnames(ctx, client)
-	if err != nil {
+	if err != nil && errors.Is(err, util.ShardingNotEnabledError) {
 		log.Errorf("Cannot get hostnames: %s", err)
 	}
 	log.Debugf("hostnames: %v", hostnames)
@@ -217,12 +223,11 @@ func main() {
 	if err != nil {
 		message := fmt.Sprintf("Cannot get host info for %q: %s", opts.Host, err.Error())
 		log.Errorf(message)
-		os.Exit(2)
+		os.Exit(cannotGetHostInfo)
 	}
 
 	if ci.ReplicaMembers, err = util.GetReplicasetMembers(ctx, clientOptions); err != nil {
 		log.Warnf("[Error] cannot get replicaset members: %v\n", err)
-		os.Exit(2)
 	}
 	log.Debugf("replicaMembers:\n%+v\n", ci.ReplicaMembers)
 
@@ -270,10 +275,9 @@ func main() {
 	out, err := formatResults(ci, opts.OutputFormat)
 	if err != nil {
 		log.Errorf("Cannot format the results: %s", err.Error())
-		os.Exit(1)
+		os.Exit(cannotFormatResults)
 	}
 	fmt.Println(string(out))
-
 }
 
 func formatResults(ci *collectedInfo, format string) ([]byte, error) {

t/pt-online-schema-change/pt-1853.t

@@ -0,0 +1,80 @@
+#!/usr/bin/env perl
+
+BEGIN {
+    die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+    unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+    unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use threads;
+use threads::shared;
+use Thread::Semaphore;
+
+use English qw(-no_match_vars);
+use Test::More;
+
+use Data::Dumper;
+use PerconaTest;
+use Sandbox;
+use SqlModes;
+use File::Temp qw/ tempdir /;
+
+require "$trunk/bin/pt-online-schema-change";
+
+plan tests => 3;
+
+my $dp = new DSNParser(opts=>$dsn_opts);
+my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
+my $master_dbh = $sb->get_dbh_for("master");
+my $master_dsn = $sb->dsn_for("master");
+
+# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
+# so we need to specify --set-vars innodb_lock_wait_timeout=3 else the
+# tool will die.
+my @args = (qw(--set-vars innodb_lock_wait_timeout=3));
+my $output;
+my $exit_status;
+
+$sb->load_file('master', "t/pt-online-schema-change/samples/pt-1853.sql");
+
+($output, $exit_status) = full_output(
+    sub { pt_online_schema_change::main(@args, "$master_dsn,D=test,t=jointit",
+            '--execute', 
+            '--alter', "engine=innodb",
+            '--alter-foreign-keys-method', 'rebuild_constraints'
+        ),
+    },
+    stderr => 1,
+);
+
+isnt(
+    $exit_status,
+    0,
+    "PT-1853, there are self-referencing FKs -> exit status != 0",
+);
+
+($output, $exit_status) = full_output(
+    sub { pt_online_schema_change::main(@args, "$master_dsn,D=test,t=jointit",
+            '--execute', 
+            '--alter', "engine=innodb",
+            '--alter-foreign-keys-method', 'rebuild_constraints',
+            '--no-check-foreign-keys'
+        ),
+    },
+    stderr => 1,
+);
+
+isnt(
+    $exit_status,
+    0,
+    "PT-1853, there are self-referencing FKs but --no-check-foreign-keys was specified -> exit status = 0",
+);
+
+# #############################################################################
+# Done.
+# #############################################################################
+$sb->wipe_clean($master_dbh);
+ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+done_testing;

t/pt-online-schema-change/rename_self_ref_fk_constraints.t

@@ -64,9 +64,9 @@ my $constraints = $master_dbh->selectall_arrayref($query);
 is_deeply(
    $constraints,
    [
-      ['person', '_fk_testId'],
-      ['test_table', '_fk_person'],
-      ['test_table', '__fk_refId'],
+      ['person', 'fk_testId'],
+      ['test_table', 'fk_person'],
+      ['test_table', 'fk_refId'],
    ],
    "First run adds or removes underscore from constraint names, accordingly"
 );
@@ -94,9 +94,9 @@ $constraints = $master_dbh->selectall_arrayref($query);
 is_deeply(
    $constraints,
    [
-      ['person', '__fk_testId'],
-      ['test_table', '_fk_refId'],
-      ['test_table', '__fk_person'],
+      ['person', 'fk_testId'],
+      ['test_table', 'fk_person'],
+      ['test_table', 'fk_refId'],
    ],
    "Second run self-referencing will be one due to rebuild_constraints"
 );

t/pt-online-schema-change/rename_self_ref_fk_constraints_pt_1802.t

@@ -60,13 +60,14 @@ my $query = <<"END";
 ORDER BY TABLE_NAME, CONSTRAINT_NAME
 END
 my $constraints = $master_dbh->selectall_arrayref($query);
+my @constraints = sort { @$a[0].@$a[1] cmp @$b[0].@$b[1] } @$constraints;
 
 is_deeply(
    $constraints,
    [
-      ['person', '_fk_testId'],
-      ['test_table', '_fk_person'],
-      ['test_table', '__fk_refId'],
+      ['person', 'fk_testId'],
+      ['test_table', 'fk_person'],
+      ['test_table', 'fk_refId'],
    ],
    "First run adds or removes underscore from constraint names, accordingly"
 );
@@ -90,13 +91,14 @@ ORDER BY TABLE_NAME, CONSTRAINT_NAME
 END
 $constraints = $master_dbh->selectall_arrayref($query);
 
+@constraints = sort { @$a[0].@$a[1] cmp @$b[0].@$b[1] } @$constraints;
 
 is_deeply(
-   $constraints,
+   \@constraints,
    [
-      ['person', '__fk_testId'],
-      ['test_table', '_fk_refId'],
-      ['test_table', '__fk_person'],
+      ['person', 'fk_testId'],
+      ['test_table', 'fk_person'],
+      ['test_table', 'fk_refId'],
    ],
    "Second run self-referencing will be one due to rebuild_constraints"
 );

t/pt-online-schema-change/samples/pt-1853.sql

@@ -0,0 +1,19 @@
+DROP DATABASE IF EXISTS test;
+CREATE DATABASE test;
+
+USE test;
+
+CREATE TABLE t1 (
+id int,
+f1 int
+);
+
+CREATE TABLE `joinit` (
+`i` int(11) NOT NULL AUTO_INCREMENT,
+`s` varchar(64) DEFAULT NULL,
+`t` time NOT NULL,
+`g` int(11) NOT NULL,
+`j` int(11) NOT NULL DEFAULT 1,
+PRIMARY KEY (`i`))
+ENGINE=InnoDB;
+ALTER TABLE joinit ADD FOREIGN KEY i_fk (j) REFERENCES joinit (i) ON UPDATE cascade ON DELETE restrict;