mirror of
https://github.com/percona/percona-toolkit.git
synced 2025-09-06 04:17:55 +00:00
Compare commits
33 Commits
PT-1979_md
...
pmm-2.33.0
Author | SHA1 | Date | |
---|---|---|---|
![]() |
873a18a589 | ||
![]() |
163346302b | ||
![]() |
f04a325b75 | ||
![]() |
fa040958ed | ||
![]() |
7fd3828ef7 | ||
![]() |
a4c454d4c5 | ||
![]() |
b8a28a2150 | ||
![]() |
f78e75d391 | ||
![]() |
d63e16f7d4 | ||
![]() |
d881738e2f | ||
![]() |
cae3c7be0a | ||
![]() |
d4dec7599b | ||
![]() |
5cdb98b487 | ||
![]() |
8519e103ad | ||
![]() |
89f60d4a94 | ||
![]() |
42bd1ed9d5 | ||
![]() |
ed6ce6b50a | ||
![]() |
a490203471 | ||
![]() |
670e1159bf | ||
![]() |
3243008c59 | ||
![]() |
9837bbdd2e | ||
![]() |
eaa95fd0e5 | ||
![]() |
957451a707 | ||
![]() |
e5cb95130d | ||
![]() |
70fd5a71b2 | ||
![]() |
c669972622 | ||
![]() |
9c00264e24 | ||
![]() |
2cb94372d5 | ||
![]() |
0bb5d6b6fb | ||
![]() |
102a3dd838 | ||
![]() |
10fbe64649 | ||
![]() |
8a4acd6201 | ||
![]() |
a5fe94e4af |
3
.gitignore
vendored
3
.gitignore
vendored
@@ -12,6 +12,7 @@ Makefile.old
|
||||
bin/pt-mongodb-*
|
||||
bin/pt-secure-*
|
||||
bin/pt-pg-*
|
||||
bin/pt-k8s-debug-collector
|
||||
!src/go/pt-mongodb-query-digest/vendor/vendor.json
|
||||
!src/go/pt-mongodb-summary/vendor/vendor.json
|
||||
src/go/pt-mongodb-query-digest/vendor/
|
||||
@@ -25,4 +26,4 @@ config/deb/control.bak
|
||||
config/rpm/percona-toolkit.spec.bak
|
||||
config/sphinx-build/percona-theme/*
|
||||
coverage.out
|
||||
.idea
|
||||
.idea
|
||||
|
21
Changelog
21
Changelog
@@ -1,21 +1,28 @@
|
||||
Changelog for Percona Toolkit
|
||||
|
||||
v3.3.2 WIP release date not set yet
|
||||
v3.4.0 release 2022-07-11
|
||||
|
||||
* New Tool PT-1978: Add reporting on unused/redundant indexes for MongoDB by pt-mongodb-summary
|
||||
* Improvement PT-1417: Inconsistent creation of toolkit tables
|
||||
* Improvement PT-1800: The environment variable PTDEBUG=1 exposes the passwords
|
||||
* Improvement PT-1940: ptsoc dropswap method that was rejected for Mysql 8 has been fixed with Mysql 8.0.14
|
||||
* Improvement PT-1979: Add gathering of admin parameters for MongoDB by pt-mongodb-summary
|
||||
* Improvement PT-2037: Add option --skip-mysql or --system-only for pt-stalk
|
||||
* Fixed bug PT-1218: pt-stalk ominous open_tables function
|
||||
* Fixed bug PT-1336: pt-stalk removes user's files from the destination directory
|
||||
* Fixed bug PT-1398: pt-stalk gets the incorrect mysqld pid when the host installed a multi MySQL instance
|
||||
* Fixed bug PT-1627: pt-mysql-summary doesn't verify which version of jemalloc is in use
|
||||
* Fixed bug PT-1747: pt-online-schema-change: metadata lock can break database for rebuild_constraints
|
||||
* Improvement PT-1800: PTDEBUG=1 exposes passwords
|
||||
* Fixed bug PT-1900: pt-query-digest not hiding parameter properly sometimes when parameter=binary
|
||||
* Improvement PT-1940: ptsoc dropswap with mysql8: revise rejection (Thanks duxthefux)
|
||||
* Fixed bug PT-1887: pt-diskstat is not working for new kernels
|
||||
* Fixed bug PT-1900: At times, pt-query-digest does not hide the parameters properly when parameter=binary
|
||||
* Fixed bug PT-1953: pt-summary typo: Memory mamagement.
|
||||
* Fixed bug PT-1959: go part of the toolkit still has the version 3.3.0
|
||||
* Fixed bug PT-1965: pt-stalk --mysql-only doesn't collect mysqladmin outputs (Thanks Sergey Kuzmichev)
|
||||
* Fixed bug PT-1965: pt-stalk --mysql-only doesn't collect mysqladmin output (Thanks Sergey Kuzmichev)
|
||||
* Fixed bug PT-1966: Test no_drop_no_swap for the pt-online-schema-change is broken (Thanks Tsubasa Tanaka)
|
||||
* Fixed bug PT-1974: Support fingerprinting for --print in pt-kill
|
||||
* Fixed bug PT-1983: pt-summary missing one DIMM
|
||||
* Admin task PT-2009: Update Go Lang dependencies
|
||||
* Admin task PT-2011: Make build reproducible
|
||||
* Fixed bug PT-2016: pt-table-checksum fails to build replace query when table lacks primary key
|
||||
* Fixed bug PT-2023: pt-upgrade Error: Wide character in print
|
||||
|
||||
v3.3.1 release 2021-04-28
|
||||
|
||||
|
@@ -2,16 +2,16 @@ use ExtUtils::MakeMaker;
|
||||
|
||||
WriteMakefile(
|
||||
NAME => 'percona-toolkit',
|
||||
VERSION => '3.3.1',
|
||||
VERSION => '3.4.0',
|
||||
EXE_FILES => [ <bin/*> ],
|
||||
MAN1PODS => {
|
||||
'docs/percona-toolkit.pod' => 'blib/man1/percona-toolkit.1p',
|
||||
map {
|
||||
(my $name = $_) =~ s/^bin.//;
|
||||
my $file_name = $_;
|
||||
if ( $file_name !~ m/mongo/ ) {
|
||||
if ( ( $file_name !~ m/mongo/ ) || ( $file_name !~ m/pg/ ) ) {
|
||||
$_ => "blib/man1/$name.1p";
|
||||
}
|
||||
}
|
||||
} <bin/*>
|
||||
},
|
||||
MAN3PODS => {}, # man(3) pages are for C libs
|
||||
|
@@ -1364,6 +1364,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-align 3.3.2
|
||||
pt-align 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -8661,6 +8661,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-archiver 3.3.2
|
||||
pt-archiver 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -5917,6 +5917,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-config-diff 3.3.2
|
||||
pt-config-diff 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -5710,6 +5710,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-deadlock-logger 3.3.2
|
||||
pt-deadlock-logger 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -5684,6 +5684,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-diskstats 3.3.2
|
||||
pt-diskstats 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -5771,6 +5771,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-duplicate-key-checker 3.3.2
|
||||
pt-duplicate-key-checker 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -1653,6 +1653,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-fifo-split 3.3.2
|
||||
pt-fifo-split 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -5132,6 +5132,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-find 3.3.2
|
||||
pt-find 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -2262,6 +2262,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-fingerprint 3.3.2
|
||||
pt-fingerprint 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -4693,6 +4693,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-fk-error-logger 3.3.2
|
||||
pt-fk-error-logger 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -7394,6 +7394,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-heartbeat 3.3.2
|
||||
pt-heartbeat 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -7705,6 +7705,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-index-usage 3.3.2
|
||||
pt-index-usage 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -1132,7 +1132,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-ioprofile 3.3.2
|
||||
pt-ioprofile 3.4.0
|
||||
|
||||
=cut
|
||||
|
||||
|
@@ -8576,6 +8576,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-kill 3.3.2
|
||||
pt-kill 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -809,7 +809,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-mext 3.3.2
|
||||
pt-mext 3.4.0
|
||||
|
||||
=cut
|
||||
|
||||
|
@@ -3296,7 +3296,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-mysql-summary 3.3.2
|
||||
pt-mysql-summary 3.4.0
|
||||
|
||||
=cut
|
||||
|
||||
|
@@ -13469,6 +13469,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-online-schema-change 3.3.2
|
||||
pt-online-schema-change 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -901,7 +901,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-pmp 3.3.2
|
||||
pt-pmp 3.4.0
|
||||
|
||||
=cut
|
||||
|
||||
|
@@ -16977,6 +16977,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-query-digest 3.3.2
|
||||
pt-query-digest 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -2618,6 +2618,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-show-grants 3.3.2
|
||||
pt-show-grants 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -1250,7 +1250,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-sift 3.3.2
|
||||
pt-sift 3.4.0
|
||||
|
||||
=cut
|
||||
|
||||
|
@@ -4993,6 +4993,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-slave-delay 3.3.2
|
||||
pt-slave-delay 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -4528,6 +4528,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-slave-find 3.3.2
|
||||
pt-slave-find 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -6164,6 +6164,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-slave-restart 3.3.2
|
||||
pt-slave-restart 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -2486,7 +2486,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-stalk 3.3.2
|
||||
pt-stalk 3.4.0
|
||||
|
||||
=cut
|
||||
|
||||
|
@@ -2729,7 +2729,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-summary 3.3.2
|
||||
pt-summary 3.4.0
|
||||
|
||||
=cut
|
||||
|
||||
|
@@ -14188,6 +14188,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-table-checksum 3.3.2
|
||||
pt-table-checksum 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -13101,6 +13101,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-table-sync 3.3.2
|
||||
pt-table-sync 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -8519,6 +8519,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-table-usage 3.3.2
|
||||
pt-table-usage 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -11454,6 +11454,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-upgrade 3.3.2
|
||||
pt-upgrade 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -6262,6 +6262,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-variable-advisor 3.3.2
|
||||
pt-variable-advisor 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -3308,6 +3308,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
pt-visual-explain 3.3.2
|
||||
pt-visual-explain 3.4.0
|
||||
|
||||
=cut
|
||||
|
@@ -48,9 +48,9 @@ copyright = u'2021, Percona LLC and/or its affiliates'
|
||||
# built documents.
|
||||
#
|
||||
# The short X.Y version.
|
||||
version = '3.3'
|
||||
version = '3.4'
|
||||
# The full version, including alpha/beta/rc tags.
|
||||
release = '3.3.1'
|
||||
release = '3.4.0'
|
||||
|
||||
# The language for content autogenerated by Sphinx. Refer to documentation
|
||||
# for a list of supported languages.
|
||||
|
@@ -567,6 +567,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
Percona Toolkit v3.3.1 released 2021-04-22
|
||||
Percona Toolkit v3.4.0 released 2022-04-05
|
||||
|
||||
=cut
|
||||
|
3
go.mod
3
go.mod
@@ -5,6 +5,7 @@ go 1.17
|
||||
require (
|
||||
github.com/Masterminds/semver v1.4.2
|
||||
github.com/alecthomas/kingpin v2.2.6+incompatible
|
||||
github.com/alecthomas/kong v0.5.0
|
||||
github.com/go-ini/ini v1.66.4
|
||||
github.com/golang/mock v1.4.4
|
||||
github.com/google/uuid v1.3.0
|
||||
@@ -40,9 +41,11 @@ require (
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.15.1 // indirect
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/rogpeppe/go-internal v1.6.1 // indirect
|
||||
github.com/smartystreets/goconvey v1.7.2 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.10 // indirect
|
||||
github.com/tklauser/numcpus v0.4.0 // indirect
|
||||
|
5
go.sum
5
go.sum
@@ -13,6 +13,9 @@ github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDO
|
||||
github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
|
||||
github.com/alecthomas/kingpin v2.2.6+incompatible h1:5svnBTFgJjZvGKyYBtMB0+m5wvrbUHiqye8wRJMlnYI=
|
||||
github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE=
|
||||
github.com/alecthomas/kong v0.5.0 h1:u8Kdw+eeml93qtMZ04iei0CFYve/WPcA5IFh+9wSskE=
|
||||
github.com/alecthomas/kong v0.5.0/go.mod h1:uzxf/HUh0tj43x1AyJROl3JT7SgsZ5m+icOv1csRhc0=
|
||||
github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRYS3XieqF+Z9B9gNxo/eANAJCF2eiN4=
|
||||
@@ -242,6 +245,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
|
||||
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/shirou/gopsutil v2.20.8+incompatible h1:8c7Atn0FAUZJo+f4wYbN0iVpdWniCQk7IYwGtgdh1mY=
|
||||
github.com/shirou/gopsutil v2.20.8+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
|
||||
|
@@ -18,7 +18,7 @@
|
||||
# ###########################################################################
|
||||
package Percona::Toolkit;
|
||||
|
||||
our $VERSION = '3.3.2';
|
||||
our $VERSION = '3.4.0';
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
|
@@ -1,7 +1,13 @@
|
||||
.PHONY: all style format build test vet tarball linux-amd64 clean help
|
||||
|
||||
help: ## Display this help message.
|
||||
@echo "Please use \`make <target>\` where <target> is one of:"
|
||||
@grep '^[a-zA-Z]' $(MAKEFILE_LIST) | \
|
||||
awk -F ':.*?## ' 'NF==2 {printf " %-26s%s\n", $$1, $$2}'
|
||||
@echo
|
||||
@echo 'To build binaries use: VERSION=m.n.p make <target>'
|
||||
@echo 'Example: VERSION=3.4.0 make linux-amd64'
|
||||
@echo 'The binaries generation uses pwd command to set the base directory. Please run make from the same directory where the Makefile is located.'
|
||||
|
||||
DATE_FMT = +%FT%T%z
|
||||
ifdef SOURCE_DATE_EPOCH
|
||||
@@ -12,17 +18,19 @@ endif
|
||||
|
||||
GO := go
|
||||
pkgs = $(shell find . -type d -name "pt-*" -exec basename {} \;)
|
||||
VERSION=$(shell git describe --abbrev=0 --tags)
|
||||
BUILD=$(BUILD_DATE)
|
||||
GOVERSION=$(shell go version | cut --delimiter=" " -f3)
|
||||
COMMIT=$(shell git rev-list -1 HEAD)
|
||||
GOUTILSDIR ?= $(GOPATH)/bin
|
||||
FILES = $(shell find . -type f -name '*.go' -not -path "./vendor/*")
|
||||
|
||||
PREFIX=$(shell pwd)
|
||||
TOP_DIR=$(shell git rev-parse --show-toplevel)
|
||||
BIN_DIR=$(shell git rev-parse --show-toplevel)/bin
|
||||
SRC_DIR=$(shell git rev-parse --show-toplevel)/src/go
|
||||
|
||||
CUR_DIR=$(shell pwd)
|
||||
COMMIT=$(shell git rev-list -1 HEAD)
|
||||
|
||||
TOP_DIR=${CUR_DIR}
|
||||
BIN_DIR=${CUR_DIR}/../../bin
|
||||
SRC_DIR=${CUR_DIR}/../../src/go
|
||||
|
||||
LDFLAGS="-X main.Version=${VERSION} -X main.Build=${BUILD} -X main.GoVersion=${GOVERSION} -X main.Commit=${COMMIT} -s -w"
|
||||
|
||||
TEST_PSMDB_VERSION?=4.0
|
||||
@@ -65,7 +73,6 @@ MINIO_SECRET_ACCESS_KEY=secret00000
|
||||
export MINIO_ACCESS_KEY_ID
|
||||
export MINIO_SECRET_ACCESS_KEY
|
||||
|
||||
.PHONY: all style format build test vet tarball linux-amd64
|
||||
|
||||
$(GOUTILSDIR)/dep:
|
||||
go get -u github.com/golang/dep/cmd/dep
|
||||
@@ -116,20 +123,20 @@ env-down: env ## Clean-up MongoDB docker containers cluster
|
||||
docker-compose down -v
|
||||
rm .env
|
||||
|
||||
linux-amd64: ## Build Mongo tools for linux-amd64
|
||||
@echo "Building linux/amd64 binaries in ${BIN_DIR}"
|
||||
linux-amd64: ## Build Mongo tools for linux-amd64.
|
||||
@echo "Building linux/amd64 binaries in ${BIN_DIR} as version ${VERSION}"
|
||||
@cd ${TOP_DIR} && go get ./...
|
||||
@$(foreach pkg,$(pkgs),rm -f ${BIN_DIR}/$(pkg) 2> /dev/null;)
|
||||
@$(foreach pkg,$(pkgs),GOOS=linux GOARCH=amd64 go build -ldflags ${LDFLAGS} -o ${BIN_DIR}/$(pkg) ./$(pkg);)
|
||||
|
||||
linux-386: ## Build Mongo tools for linux-386
|
||||
@echo "Building linux/386 binaries in ${BIN_DIR}"
|
||||
@echo "Building linux/386 binaries in ${BIN_DIR} as version ${VERSION}"
|
||||
@cd ${TOP_DIR} && go get ./...
|
||||
@$(foreach pkg,$(pkgs),rm -f ${BIN_DIR}/$(pkg) 2> /dev/null;)
|
||||
@$(foreach pkg,$(pkgs),GOOS=linux GOARCH=386 go build -ldflags ${LDFLAGS} -o ${BIN_DIR}/$(pkg) ./$(pkg);)
|
||||
|
||||
darwin-amd64: ## Build Mongo tools for darwin-amd64 (MacOS)
|
||||
@echo "Building darwin/amd64 binaries in ${BIN_DIR}"
|
||||
@echo "Building darwin/amd64 binaries in ${BIN_DIR} as version ${VERSION}"
|
||||
@cd ${TOP_DIR} && go get ./...
|
||||
@$(foreach pkg,$(pkgs),rm -f ${BIN_DIR}/$(pkg) 2> /dev/null;)
|
||||
@$(foreach pkg,$(pkgs),GOOS=darwin GOARCH=amd64 go build -ldflags ${LDFLAGS} -o ${BIN_DIR}/$(pkg) ./$(pkg);)
|
||||
|
@@ -1,22 +1,21 @@
|
||||
package proto
|
||||
|
||||
type ShardStas struct {
|
||||
Ns string `json:"ns"`
|
||||
Count int64 `json:"count"`
|
||||
Size int64 `json:"size"`
|
||||
AvgObjSize int64 `json:"avgObjSize"`
|
||||
NumExtents int64 `json:"numExtents"`
|
||||
StorageSize int64 `json:"storageSize"`
|
||||
LastExtentSize int64 `json:"lastExtentSize"`
|
||||
PaddingFactor int64 `json:"paddingFactor"`
|
||||
PaddingFactorNote string `json:"paddingFactorNote"`
|
||||
UserFlags int64 `json:"userFlags"`
|
||||
Capped bool `json:"capped"`
|
||||
Nindexes int64 `json:"nindexes"`
|
||||
IndexDetails struct {
|
||||
} `json:"indexDetails"`
|
||||
TotalIndexSize int64 `json:"totalIndexSize"`
|
||||
IndexSizes struct {
|
||||
Ns string `json:"ns"`
|
||||
Count int64 `json:"count"`
|
||||
Size int64 `json:"size"`
|
||||
AvgObjSize int64 `json:"avgObjSize"`
|
||||
NumExtents int64 `json:"numExtents"`
|
||||
StorageSize int64 `json:"storageSize"`
|
||||
LastExtentSize int64 `json:"lastExtentSize"`
|
||||
PaddingFactor int64 `json:"paddingFactor"`
|
||||
PaddingFactorNote string `json:"paddingFactorNote"`
|
||||
UserFlags int64 `json:"userFlags"`
|
||||
Capped bool `json:"capped"`
|
||||
Nindexes int64 `json:"nindexes"`
|
||||
IndexDetails struct{} `json:"indexDetails"`
|
||||
TotalIndexSize int64 `json:"totalIndexSize"`
|
||||
IndexSizes struct {
|
||||
ID int64 `json:"_id_"`
|
||||
IDHashed int64 `json:"_id_hashed"`
|
||||
} `json:"indexSizes"`
|
||||
|
@@ -72,7 +72,7 @@ func New(location, namespace, resource string) Dumper {
|
||||
cmd: "kubectl",
|
||||
resources: resources,
|
||||
location: "cluster-dump",
|
||||
mode: int64(0777),
|
||||
mode: int64(0o777),
|
||||
namespace: namespace,
|
||||
crType: resource,
|
||||
}
|
||||
|
48
src/go/pt-mongodb-index-check/README.md
Normal file
48
src/go/pt-mongodb-index-check/README.md
Normal file
@@ -0,0 +1,48 @@
|
||||
# pt-mongodb-index-check
|
||||
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
This tool can perform checks on MongoDB indexes.
|
||||
|
||||
Currently, these checks are available:
|
||||
|
||||
### Duplicated indexes
|
||||
|
||||
Check for indexes that are the prefix of other indexes. For example if we have these 2 indexes
|
||||
|
||||
```
|
||||
db.getSiblingDB("testdb").test_col.createIndex({"f1": 1, "f2": -1, "f3": 1, "f4": 1}, {"name": "idx_01"});
|
||||
db.getSiblingDB("testdb").test_col.createIndex({"f1": 1, "f2": -1, "f3": 1}, {"name": "idx_02"});
|
||||
```
|
||||
|
||||
The index `idx_02` is the prefix of `idx_01` because it has the same keys in the same order so, `idx_02` can be dropped.
|
||||
|
||||
### Unused indexes.
|
||||
|
||||
This check gets the `$indexstats` for all indexes and reports those having `accesses.ops` = 0.
|
||||
|
||||
## Usage
|
||||
|
||||
Run the program as `pt-mongodb-index-check <command> [flags]`
|
||||
|
||||
#### Available commands
|
||||
|
||||
| Command | Description |
|
||||
| ---------------- | ---------------------------------- |
|
||||
| check-duplicated | Run checks for duplicated indexes. |
|
||||
| check-unused | Run check for unused indexes. |
|
||||
| check-all | Run all checks |
|
||||
|
||||
#### Available flags
|
||||
|
||||
| Flag | Description |
|
||||
| ---- | ----------- |
|
||||
|--all-databases|Check in all databases excluding system dbs.|
|
||||
|--databases=DATABASES,...|Comma separated list of databases to check.|
|
||||
|--all-collections|Check in all collections in the selected databases.|
|
||||
|--collections=COLLECTIONS,...|Comma separated list of collections to check.|
|
||||
|--mongodb.uri=<connection string>|Connection URI|
|
||||
|--json|Show output as JSON|
|
||||
|
103
src/go/pt-mongodb-index-check/indexes/duplicated.go
Normal file
103
src/go/pt-mongodb-index-check/indexes/duplicated.go
Normal file
@@ -0,0 +1,103 @@
|
||||
package indexes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
)
|
||||
|
||||
type collectionIndex struct {
|
||||
Name string `bson:"name"`
|
||||
Namespace string `bson:"ns"`
|
||||
V int `bson:"v"`
|
||||
Key primitive.D `bson:"key"`
|
||||
}
|
||||
|
||||
func (di collectionIndex) ComparableKey() string {
|
||||
str := ""
|
||||
for _, elem := range di.Key {
|
||||
str += sign(elem) + elem.Key
|
||||
}
|
||||
return str
|
||||
}
|
||||
|
||||
func sign(elem primitive.E) string {
|
||||
sign := "+"
|
||||
switch elem.Value.(type) {
|
||||
case int32: // internal MongoDB indexes like _id_ or lastUsed have the sign field as int32.
|
||||
if elem.Value.(int32) < 0 {
|
||||
sign = "-"
|
||||
}
|
||||
case float64: // All other indexes have the sign field as float64.
|
||||
if elem.Value.(float64) < 0 {
|
||||
sign = "-"
|
||||
}
|
||||
}
|
||||
return sign
|
||||
}
|
||||
|
||||
// IndexKey holds the list of fields that are part of an index, along with the field order.
|
||||
type IndexKey []primitive.E
|
||||
|
||||
// String returns the index fields as a string. The + sign means ascending on this field
|
||||
// and a - sign indicates a descending order for that field.
|
||||
func (di IndexKey) String() string {
|
||||
str := ""
|
||||
for _, elem := range di {
|
||||
str += sign(elem) + elem.Key + " "
|
||||
}
|
||||
|
||||
return str
|
||||
}
|
||||
|
||||
// DuplicateIndex represents a duplicated index pair.
|
||||
// An index is considered as the duplicate of another one if it is it's prefix.
|
||||
// Example: the index +f1-f2 is the prefix of +f1-f2+f3.
|
||||
type Duplicate struct {
|
||||
Namespace string
|
||||
Name string
|
||||
Key IndexKey
|
||||
ContainerName string
|
||||
ContainerKey IndexKey
|
||||
}
|
||||
|
||||
func FindDuplicated(ctx context.Context, client *mongo.Client, database, collection string) ([]Duplicate, error) {
|
||||
di := []Duplicate{}
|
||||
|
||||
cursor, err := client.Database(database).Collection(collection).Indexes().List(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var results []collectionIndex
|
||||
if err = cursor.All(context.TODO(), &results); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
sort.Slice(results, func(i, j int) bool {
|
||||
return results[i].ComparableKey() < results[j].ComparableKey()
|
||||
})
|
||||
|
||||
for i := 0; i < len(results)-1; i++ {
|
||||
for j := i + 1; j < len(results); j++ {
|
||||
if strings.HasPrefix(results[j].ComparableKey(), results[i].ComparableKey()) {
|
||||
idx := Duplicate{
|
||||
Namespace: database + "." + collection,
|
||||
Name: results[i].Name,
|
||||
Key: make([]primitive.E, len(results[i].Key)),
|
||||
ContainerName: results[j].Name,
|
||||
ContainerKey: make([]primitive.E, len(results[j].Key)),
|
||||
}
|
||||
copy(idx.Key, results[i].Key)
|
||||
copy(idx.ContainerKey, results[j].Key)
|
||||
di = append(di, idx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return di, nil
|
||||
}
|
117
src/go/pt-mongodb-index-check/indexes/duplicated_test.go
Normal file
117
src/go/pt-mongodb-index-check/indexes/duplicated_test.go
Normal file
@@ -0,0 +1,117 @@
|
||||
package indexes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/AlekSi/pointer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"gopkg.in/mgo.v2/bson"
|
||||
|
||||
tu "github.com/percona/percona-toolkit/src/go/internal/testutils"
|
||||
)
|
||||
|
||||
func TestDuplicateIndexes(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
client, err := tu.TestClient(ctx, tu.MongoDBShard1PrimaryPort)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot get a new MongoDB client: %s", err)
|
||||
}
|
||||
|
||||
dbname := "test_db"
|
||||
collname := "test_col"
|
||||
|
||||
database := client.Database(dbname)
|
||||
database.Drop(ctx) //nolint:errcheck
|
||||
defer database.Drop(ctx) //nolint:errcheck
|
||||
|
||||
_, err = database.Collection(collname).InsertOne(ctx, bson.M{"f1": 1, "f2": "2", "f3": "a", "f4": "c"})
|
||||
assert.NoError(t, err)
|
||||
|
||||
testCases := []primitive.D{
|
||||
{{"f1", 1}, {"f2", -1}, {"f3", 1}, {"f4", 1}},
|
||||
{{"f1", 1}, {"f2", -1}, {"f3", 1}, {"f4", 1}}, // this will throw a duplicate index error
|
||||
{{"f1", 1}, {"f2", -1}, {"f3", 1}},
|
||||
{{"f1", 1}, {"f2", -1}},
|
||||
{{"f3", -1}},
|
||||
}
|
||||
|
||||
errCount := 0
|
||||
for i, tc := range testCases {
|
||||
mod := mongo.IndexModel{
|
||||
Keys: tc,
|
||||
Options: &options.IndexOptions{
|
||||
Name: pointer.ToString(fmt.Sprintf("idx_%02d", i)),
|
||||
},
|
||||
}
|
||||
_, err := database.Collection(collname).Indexes().CreateOne(ctx, mod)
|
||||
if err != nil {
|
||||
errCount++
|
||||
}
|
||||
}
|
||||
/*
|
||||
At this point we have 5 indexes: _id: (MongoDB's default), idx_00, idx_02, idx_03, idx_04.
|
||||
idx_01 wasn't created since it duplicates idx_00 and errCount=1.
|
||||
*/
|
||||
|
||||
assert.Equal(t, 1, errCount)
|
||||
|
||||
want := []Duplicate{
|
||||
{
|
||||
Name: "idx_03",
|
||||
Namespace: "test_db.test_col",
|
||||
Key: IndexKey{
|
||||
{Key: "f1", Value: int32(1)},
|
||||
{Key: "f2", Value: int32(-1)},
|
||||
},
|
||||
ContainerName: "idx_02",
|
||||
ContainerKey: IndexKey{
|
||||
{Key: "f1", Value: int32(1)},
|
||||
{Key: "f2", Value: int32(-1)},
|
||||
{Key: "f3", Value: int32(1)},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "idx_03",
|
||||
Namespace: "test_db.test_col",
|
||||
Key: IndexKey{
|
||||
{Key: "f1", Value: int32(1)},
|
||||
{Key: "f2", Value: int32(-1)},
|
||||
},
|
||||
ContainerName: "idx_00",
|
||||
ContainerKey: IndexKey{
|
||||
{Key: "f1", Value: int32(1)},
|
||||
{Key: "f2", Value: int32(-1)},
|
||||
{Key: "f3", Value: int32(1)},
|
||||
{Key: "f4", Value: int32(1)},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "idx_02",
|
||||
Namespace: "test_db.test_col",
|
||||
Key: IndexKey{
|
||||
{Key: "f1", Value: int32(1)},
|
||||
{Key: "f2", Value: int32(-1)},
|
||||
{Key: "f3", Value: int32(1)},
|
||||
},
|
||||
ContainerName: "idx_00",
|
||||
ContainerKey: IndexKey{
|
||||
{Key: "f1", Value: int32(1)},
|
||||
{Key: "f2", Value: int32(-1)},
|
||||
{Key: "f3", Value: int32(1)},
|
||||
{Key: "f4", Value: int32(1)},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
di, err := FindDuplicated(ctx, client, dbname, collname)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, want, di)
|
||||
}
|
62
src/go/pt-mongodb-index-check/indexes/unused.go
Normal file
62
src/go/pt-mongodb-index-check/indexes/unused.go
Normal file
@@ -0,0 +1,62 @@
|
||||
package indexes
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"gopkg.in/mgo.v2/bson"
|
||||
)
|
||||
|
||||
var systemDBs = []string{"admin", "config", "local", "system.profile"} //nolint:gochecknoglobals
|
||||
// IndexStat hold an index usage statistics.
|
||||
type IndexStat struct {
|
||||
Accesses struct {
|
||||
Ops int64 `bson:"ops"`
|
||||
Since primitive.DateTime `bson:"since"`
|
||||
} `bson:"accesses"`
|
||||
Spec struct {
|
||||
Name string `bson:"name"`
|
||||
Namespace string `bson:"ns"`
|
||||
V int32 `bson:"v"`
|
||||
Key primitive.D `bson:"key"`
|
||||
} `bson:"spec"`
|
||||
Name string `bson:"name"`
|
||||
Key primitive.D `bson:"key"`
|
||||
Host string `bson:"host"`
|
||||
}
|
||||
|
||||
func in(search string, items []string) bool {
|
||||
for _, item := range items {
|
||||
if search == item {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FindUnusedIndexes returns a list of unused indexes for the given database and collection.
|
||||
func FindUnused(ctx context.Context, client *mongo.Client, database, collection string) ([]IndexStat, error) {
|
||||
aggregation := mongo.Pipeline{
|
||||
{{Key: "$indexStats", Value: primitive.M{}}},
|
||||
{{Key: "$match", Value: primitive.M{"accesses.ops": 0}}},
|
||||
{{Key: "$match", Value: primitive.M{"name": bson.M{"$ne": "_id_"}}}},
|
||||
}
|
||||
|
||||
if in(database, systemDBs) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
cursor, err := client.Database(database).Collection(collection).Aggregate(ctx, aggregation)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannot run $indexStats for unused indexes")
|
||||
}
|
||||
|
||||
var stats []IndexStat
|
||||
if err = cursor.All(ctx, &stats); err != nil {
|
||||
return nil, errors.Wrap(err, "cannot get $indexStats for unused indexes")
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
82
src/go/pt-mongodb-index-check/indexes/unused_test.go
Normal file
82
src/go/pt-mongodb-index-check/indexes/unused_test.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package indexes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/AlekSi/pointer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"gopkg.in/mgo.v2/bson"
|
||||
|
||||
tu "github.com/percona/percona-toolkit/src/go/internal/testutils"
|
||||
)
|
||||
|
||||
func TestUnusedIndexes(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
client, err := tu.TestClient(ctx, tu.MongoDBShard1PrimaryPort)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot get a new MongoDB client: %s", err)
|
||||
}
|
||||
|
||||
dbname := "test_db"
|
||||
collname := "test_col"
|
||||
|
||||
database := client.Database(dbname)
|
||||
database.Drop(ctx) //nolint:errcheck
|
||||
defer database.Drop(ctx) //nolint:errcheck
|
||||
|
||||
testCases := []primitive.D{
|
||||
{{"f1", 1}, {"f2", -1}, {"f3", 1}, {"f4", 1}},
|
||||
{{"f3", -1}},
|
||||
{{"f4", -1}},
|
||||
}
|
||||
|
||||
errCount := 0
|
||||
for i, tc := range testCases {
|
||||
mod := mongo.IndexModel{
|
||||
Keys: tc,
|
||||
Options: &options.IndexOptions{
|
||||
Name: pointer.ToString(fmt.Sprintf("idx_%02d", i)),
|
||||
},
|
||||
}
|
||||
_, err := database.Collection(collname).Indexes().CreateOne(ctx, mod)
|
||||
if err != nil {
|
||||
errCount++
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
_, err = database.Collection(collname).InsertOne(ctx,
|
||||
bson.M{"f1": rand.Int63n(1000), "f2": rand.Int63n(1000), "f3": rand.Int63n(1000), "f4": rand.Int63n(1000)})
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
// Make use of idx_02: {"f4": -1} to exclude it from the results so we ensure only unused indexes
|
||||
// are being listed.
|
||||
_, err = database.Collection(collname).Find(ctx, primitive.M{"f4": primitive.M{"$gt": 500}})
|
||||
assert.NoError(t, err)
|
||||
|
||||
want := []string{"idx_00", "idx_01"}
|
||||
|
||||
ui, err := FindUnused(ctx, client, dbname, collname)
|
||||
assert.NoError(t, err)
|
||||
|
||||
got := make([]string, 0, len(ui))
|
||||
for _, idx := range ui {
|
||||
// compare only names because the index struct has a timestamp in it and it is variable.
|
||||
got = append(got, idx.Name)
|
||||
}
|
||||
|
||||
sort.Strings(got)
|
||||
|
||||
assert.Equal(t, want, got)
|
||||
}
|
188
src/go/pt-mongodb-index-check/main.go
Normal file
188
src/go/pt-mongodb-index-check/main.go
Normal file
@@ -0,0 +1,188 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/alecthomas/kong"
|
||||
"github.com/pkg/errors"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
|
||||
"github.com/percona/percona-toolkit/src/go/pt-mongodb-index-check/indexes"
|
||||
"github.com/percona/percona-toolkit/src/go/pt-mongodb-index-check/templates"
|
||||
)
|
||||
|
||||
type cmdlineArgs struct {
|
||||
CheckUnused struct{} `cmd:"" name:"check-unused" help:"Check for unused indexes."`
|
||||
CheckDuplicated struct{} `cmd:"" name:"check-duplicates" help:"Check for duplicated indexes."`
|
||||
CheckAll struct{} `cmd:"" name:"check-all" help:"Check for unused and duplicated indexes."`
|
||||
ShowHelp struct{} `cmd:"" default:"1"`
|
||||
Version struct{} `cmd:"" name:"version"`
|
||||
|
||||
AllDatabases bool `name:"all-databases" xor:"db" help:"Check in all databases excluding system dbs"`
|
||||
Databases []string `name:"databases" xor:"db" help:"Comma separated list of databases to check"`
|
||||
|
||||
AllCollections bool `name:"all-collections" xor:"colls" help:"Check in all collections in the selected databases."`
|
||||
Collections []string `name:"collections" xor:"colls" help:"Comma separated list of collections to check"`
|
||||
URI string `name:"mongodb.uri" required:"" placeholder:"mongodb://host:port/admindb?options" help:"Connection URI"`
|
||||
JSON bool `name:"json" help:"Show output as JSON"`
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Unused []indexes.IndexStat
|
||||
Duplicated []indexes.Duplicate
|
||||
}
|
||||
|
||||
const (
|
||||
TOOLNAME = "pt-mongodb-index-check"
|
||||
)
|
||||
|
||||
var (
|
||||
Build string = "2020-04-23" //nolint
|
||||
GoVersion string = "1.14.1" //nolint
|
||||
Version string = "3.4.0" //nolint
|
||||
Commit string //nolint
|
||||
)
|
||||
|
||||
func main() {
|
||||
var args cmdlineArgs
|
||||
kongctx := kong.Parse(&args, kong.UsageOnError())
|
||||
|
||||
if kongctx.Command() == "version" {
|
||||
fmt.Println(TOOLNAME)
|
||||
fmt.Printf("Version %s\n", Version)
|
||||
fmt.Printf("Build: %s using %s\n", Build, GoVersion)
|
||||
fmt.Printf("Commit: %s\n", Commit)
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if !strings.HasPrefix(args.URI, "mongodb") && !strings.HasPrefix(args.URI, "mongodb+srv") {
|
||||
args.URI = "mongodb://" + args.URI
|
||||
}
|
||||
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(args.URI))
|
||||
if err != nil {
|
||||
log.Fatalf("Cannot connect to the database: %q", err)
|
||||
}
|
||||
|
||||
if args.AllDatabases {
|
||||
args.Databases, err = client.ListDatabaseNames(context.TODO(), primitive.D{})
|
||||
if err != nil {
|
||||
log.Fatalf("cannot list all databases: %s", err)
|
||||
}
|
||||
}
|
||||
if args.AllCollections {
|
||||
args.Collections = nil
|
||||
}
|
||||
|
||||
resp := response{}
|
||||
|
||||
switch kongctx.Command() {
|
||||
case "check-unused":
|
||||
resp.Unused = findUnused(ctx, client, args.Databases, args.Collections)
|
||||
case "check-duplicates":
|
||||
resp.Duplicated = findDuplicated(ctx, client, args.Databases, args.Collections)
|
||||
case "check-all":
|
||||
resp.Unused = findUnused(ctx, client, args.Databases, args.Collections)
|
||||
resp.Duplicated = findDuplicated(ctx, client, args.Databases, args.Collections)
|
||||
default:
|
||||
kong.DefaultHelpPrinter(kong.HelpOptions{}, kongctx)
|
||||
}
|
||||
|
||||
fmt.Println(output(resp, args.JSON))
|
||||
}
|
||||
|
||||
func output(resp response, asJson bool) string {
|
||||
if asJson {
|
||||
jsonStr, err := json.MarshalIndent(resp, "", "\t")
|
||||
if err != nil {
|
||||
log.Fatal("cannot encode the response as json")
|
||||
}
|
||||
return string(jsonStr)
|
||||
}
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
|
||||
t := template.Must(template.New("duplicated").Parse(templates.Duplicated))
|
||||
if err := t.Execute(buf, resp.Duplicated); err != nil {
|
||||
log.Fatal(errors.Wrap(err, "cannot parse clusterwide section of the output template"))
|
||||
}
|
||||
|
||||
t = template.Must(template.New("unused").Parse(templates.Unused))
|
||||
if err := t.Execute(buf, resp.Unused); err != nil {
|
||||
log.Fatal(errors.Wrap(err, "cannot parse clusterwide section of the output template"))
|
||||
}
|
||||
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
func findUnused(ctx context.Context, client *mongo.Client, databases []string, collections []string) []indexes.IndexStat {
|
||||
unused := []indexes.IndexStat{}
|
||||
var err error
|
||||
|
||||
colls := make([]string, len(collections))
|
||||
copy(colls, collections)
|
||||
|
||||
for _, database := range databases {
|
||||
if len(collections) == 0 {
|
||||
colls, err = client.Database(database).ListCollectionNames(ctx, primitive.D{})
|
||||
if err != nil {
|
||||
log.Errorf("cannot get the list of collections for the database %s", database)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
for _, collection := range colls {
|
||||
idx, err := indexes.FindUnused(ctx, client, database, collection)
|
||||
if err != nil {
|
||||
log.Errorf("error while checking unused indexes in %s.%s: %s", database, collection, err)
|
||||
continue
|
||||
}
|
||||
|
||||
unused = append(unused, idx...)
|
||||
}
|
||||
}
|
||||
|
||||
return unused
|
||||
}
|
||||
|
||||
func findDuplicated(ctx context.Context, client *mongo.Client, databases []string, collections []string) []indexes.Duplicate {
|
||||
duplicated := []indexes.Duplicate{}
|
||||
var err error
|
||||
|
||||
colls := make([]string, len(collections))
|
||||
copy(colls, collections)
|
||||
|
||||
for _, database := range databases {
|
||||
if len(collections) == 0 {
|
||||
colls, err = client.Database(database).ListCollectionNames(ctx, primitive.D{})
|
||||
if err != nil {
|
||||
log.Errorf("cannot get the list of collections for the database %s", database)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
for _, collection := range colls {
|
||||
dups, err := indexes.FindDuplicated(ctx, client, database, collection)
|
||||
if err != nil {
|
||||
log.Errorf("error while checking duplicated indexes in %s.%s: %s", database, collection, err)
|
||||
continue
|
||||
}
|
||||
|
||||
duplicated = append(duplicated, dups...)
|
||||
}
|
||||
}
|
||||
|
||||
return duplicated
|
||||
}
|
10
src/go/pt-mongodb-index-check/templates/duplicated.go
Normal file
10
src/go/pt-mongodb-index-check/templates/duplicated.go
Normal file
@@ -0,0 +1,10 @@
|
||||
package templates
|
||||
|
||||
// {{if $i}},{{end}} adds a comma after the first element.
|
||||
// When $i == 0 (first element) {{ if $i }} returns false (0)
|
||||
|
||||
var Duplicated = `
|
||||
Duplicated indexes
|
||||
{{ range . }}
|
||||
{{ .Namespace }}, index '{{ .Name }}', with fields { {{- range $i, $val := .Key }}{{if $i}}, {{end}}{{ $val.Key }}:{{ $val.Value }}{{ end -}} } is the prefix of '{{ .ContainerName }}' with fields { {{- range $i, $val := .ContainerKey }}{{if $i}}, {{end}}{{ $val.Key }}:{{ $val.Value }}{{ end -}} }{{ end}}
|
||||
`
|
10
src/go/pt-mongodb-index-check/templates/unused.go
Normal file
10
src/go/pt-mongodb-index-check/templates/unused.go
Normal file
@@ -0,0 +1,10 @@
|
||||
package templates
|
||||
|
||||
// {{if $i}},{{end}} adds a comma after the first element.
|
||||
// When $i == 0 (first element) {{ if $i }} returns false (0)
|
||||
|
||||
var Unused = `
|
||||
Unused indexes since last restart
|
||||
{{ range . }}
|
||||
{{ .Spec.Namespace }}, index '{{ .Name }}' with fields { {{- range $i, $val := .Key }}{{if $i}}, {{end}}{{ $val.Key }}:{{ $val.Value }} }{{ end }}{{ end}}
|
||||
`
|
@@ -13,6 +13,11 @@ import (
|
||||
|
||||
"github.com/howeyc/gopass"
|
||||
"github.com/pborman/getopt"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
|
||||
"github.com/percona/percona-toolkit/src/go/lib/config"
|
||||
"github.com/percona/percona-toolkit/src/go/lib/versioncheck"
|
||||
"github.com/percona/percona-toolkit/src/go/mongolib/fingerprinter"
|
||||
@@ -21,10 +26,6 @@ import (
|
||||
"github.com/percona/percona-toolkit/src/go/mongolib/stats"
|
||||
"github.com/percona/percona-toolkit/src/go/mongolib/util"
|
||||
"github.com/percona/percona-toolkit/src/go/pt-mongodb-query-digest/filter"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -40,7 +41,7 @@ const (
|
||||
var (
|
||||
Build string = "2020-04-23" //nolint
|
||||
GoVersion string = "1.14.1" //nolint
|
||||
Version string = "3.3.2" //nolint
|
||||
Version string = "3.4.0" //nolint
|
||||
Commit string //nolint
|
||||
)
|
||||
|
||||
|
@@ -19,18 +19,19 @@ import (
|
||||
version "github.com/hashicorp/go-version"
|
||||
"github.com/howeyc/gopass"
|
||||
"github.com/pborman/getopt"
|
||||
"github.com/percona/percona-toolkit/src/go/lib/config"
|
||||
"github.com/percona/percona-toolkit/src/go/lib/versioncheck"
|
||||
"github.com/percona/percona-toolkit/src/go/mongolib/proto"
|
||||
"github.com/percona/percona-toolkit/src/go/mongolib/util"
|
||||
"github.com/percona/percona-toolkit/src/go/pt-mongodb-summary/oplog"
|
||||
"github.com/percona/percona-toolkit/src/go/pt-mongodb-summary/templates"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/shirou/gopsutil/process"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
|
||||
"github.com/percona/percona-toolkit/src/go/lib/config"
|
||||
"github.com/percona/percona-toolkit/src/go/lib/versioncheck"
|
||||
"github.com/percona/percona-toolkit/src/go/mongolib/proto"
|
||||
"github.com/percona/percona-toolkit/src/go/mongolib/util"
|
||||
"github.com/percona/percona-toolkit/src/go/pt-mongodb-summary/oplog"
|
||||
"github.com/percona/percona-toolkit/src/go/pt-mongodb-summary/templates"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -56,7 +57,7 @@ const (
|
||||
var (
|
||||
Build string = "2020-04-23"
|
||||
GoVersion string = "1.14.1"
|
||||
Version string = "3.3.2"
|
||||
Version string = "3.4.0"
|
||||
Commit string
|
||||
|
||||
defaultConnectionTimeout = 3 * time.Second
|
||||
@@ -615,7 +616,8 @@ func getNodeType(ctx context.Context, client *mongo.Client) (string, error) {
|
||||
}
|
||||
|
||||
func getOpCountersStats(ctx context.Context, client *mongo.Client, count int,
|
||||
sleep time.Duration) (*opCounters, error) {
|
||||
sleep time.Duration,
|
||||
) (*opCounters, error) {
|
||||
oc := &opCounters{}
|
||||
prevOpCount := &opCounters{}
|
||||
ss := proto.ServerStatus{}
|
||||
|
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/pborman/getopt"
|
||||
|
||||
tu "github.com/percona/percona-toolkit/src/go/internal/testutils"
|
||||
"github.com/percona/percona-toolkit/src/go/mongolib/proto"
|
||||
)
|
||||
|
@@ -6,12 +6,13 @@ import (
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/percona/percona-toolkit/src/go/mongolib/proto"
|
||||
"github.com/percona/percona-toolkit/src/go/mongolib/util"
|
||||
"github.com/pkg/errors"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
|
||||
"github.com/percona/percona-toolkit/src/go/mongolib/proto"
|
||||
"github.com/percona/percona-toolkit/src/go/mongolib/util"
|
||||
)
|
||||
|
||||
func GetOplogInfo(ctx context.Context, hostnames []string, co *options.ClientOptions) ([]proto.OplogInfo, error) {
|
||||
|
@@ -8,20 +8,20 @@ import (
|
||||
"text/template"
|
||||
|
||||
"github.com/alecthomas/kingpin"
|
||||
_ "github.com/lib/pq"
|
||||
"github.com/pkg/errors"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/percona/percona-toolkit/src/go/lib/pginfo"
|
||||
"github.com/percona/percona-toolkit/src/go/pt-pg-summary/templates"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
var (
|
||||
Build string = "2020-04-23" //nolint
|
||||
Commit string //nolint
|
||||
GoVersion string = "1.14.1" //nolint
|
||||
Version string = "3.3.1" //nolint
|
||||
Version string = "3.4.0" //nolint
|
||||
)
|
||||
|
||||
type connOpts struct {
|
||||
|
@@ -5,10 +5,10 @@ import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/percona/percona-toolkit/src/go/lib/pginfo"
|
||||
"github.com/percona/percona-toolkit/src/go/pt-pg-summary/internal/tu"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type Test struct {
|
||||
|
@@ -16,10 +16,11 @@ import (
|
||||
"time"
|
||||
|
||||
shellwords "github.com/mattn/go-shellwords"
|
||||
"github.com/percona/percona-toolkit/src/go/pt-secure-collect/sanitize"
|
||||
"github.com/percona/percona-toolkit/src/go/pt-secure-collect/sanitize/util"
|
||||
"github.com/pkg/errors"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/percona/percona-toolkit/src/go/pt-secure-collect/sanitize"
|
||||
"github.com/percona/percona-toolkit/src/go/pt-secure-collect/sanitize/util"
|
||||
)
|
||||
|
||||
func collectData(opts *cliOptions) error {
|
||||
|
@@ -15,8 +15,8 @@ import (
|
||||
"github.com/alecthomas/kingpin"
|
||||
"github.com/go-ini/ini"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/sirupsen/logrus"
|
||||
"golang.org/x/crypto/ssh/terminal"
|
||||
)
|
||||
|
||||
@@ -86,7 +86,7 @@ var (
|
||||
|
||||
Build string = "2020-04-23" //nolint
|
||||
GoVersion string = "1.14.1" //nolint
|
||||
Version string = "3.3.1" //nolint
|
||||
Version string = "3.4.0" //nolint
|
||||
Commit string //nolint
|
||||
)
|
||||
|
||||
|
@@ -3,9 +3,10 @@ package main
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/percona/percona-toolkit/src/go/pt-secure-collect/sanitize"
|
||||
"github.com/percona/percona-toolkit/src/go/pt-secure-collect/sanitize/util"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
func sanitizeFile(opts *cliOptions) error {
|
||||
|
Reference in New Issue
Block a user