mirror of
https://github.com/percona/percona-toolkit.git
synced 2025-09-21 11:30:24 +00:00
PT-1501 New tool pt-secure-collect
New tool to collect and sanitize pt-tools outputs
This commit is contained in:
Carlos Salguero
@@ -9,6 +9,8 @@ v3.0.9
|
|||||||
* Feature PT-1509 : Only set binlog_format when necessary (Thanks Moritz Lenz)
|
* Feature PT-1509 : Only set binlog_format when necessary (Thanks Moritz Lenz)
|
||||||
* Feature PT-1508 : Adding --read-only-interval flag, and read-only check on wake-up (Thanks Shlomi Noach)
|
* Feature PT-1508 : Adding --read-only-interval flag, and read-only check on wake-up (Thanks Shlomi Noach)
|
||||||
* Improvement PT-1507 : pt-summary does not reliably read in the transparent huge pages setting (Thanks Nick Veenhof)
|
* Improvement PT-1507 : pt-summary does not reliably read in the transparent huge pages setting (Thanks Nick Veenhof)
|
||||||
|
* New tool PT-1501 : pt-secure-collect - New tool to collect and sanitize pt-tools outputs
|
||||||
|
* Improvement PT-1488 : pt-show-grants support for MySQL 8.0
|
||||||
* Feature PT-243 : Adding --max-hostname-length and --max-line-length to pt-query-digest
|
* Feature PT-243 : Adding --max-hostname-length and --max-line-length to pt-query-digest
|
||||||
|
|
||||||
v3.0.8 released 2018-03-13
|
v3.0.8 released 2018-03-13
|
||||||
|
|||||||
120
Gopkg.lock
generated
120
Gopkg.lock
generated
@@ -4,14 +4,41 @@
|
|||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/Masterminds/semver"
|
name = "github.com/Masterminds/semver"
|
||||||
packages = ["."]
|
packages = ["."]
|
||||||
revision = "15d8430ab86497c5c0da827b748823945e1cf1e1"
|
revision = "c7af12943936e8c39859482e61f0574c2fd7fc75"
|
||||||
version = "v1.4.0"
|
version = "v1.4.2"
|
||||||
|
|
||||||
|
[[projects]]
|
||||||
|
name = "github.com/Sirupsen/logrus"
|
||||||
|
packages = ["."]
|
||||||
|
revision = "c155da19408a8799da419ed3eeb0cb5db0ad5dbc"
|
||||||
|
version = "v1.0.5"
|
||||||
|
|
||||||
|
[[projects]]
|
||||||
|
name = "github.com/StackExchange/wmi"
|
||||||
|
packages = ["."]
|
||||||
|
revision = "5d049714c4a64225c3c79a7cf7d02f7fb5b96338"
|
||||||
|
version = "1.0.0"
|
||||||
|
|
||||||
|
[[projects]]
|
||||||
|
name = "github.com/alecthomas/kingpin"
|
||||||
|
packages = ["."]
|
||||||
|
revision = "947dcec5ba9c011838740e680966fd7087a71d0d"
|
||||||
|
version = "v2.2.6"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "github.com/StackExchange/wmi"
|
name = "github.com/alecthomas/template"
|
||||||
|
packages = [
|
||||||
|
".",
|
||||||
|
"parse"
|
||||||
|
]
|
||||||
|
revision = "a0175ee3bccc567396460bf5acd36800cb10c49c"
|
||||||
|
|
||||||
|
[[projects]]
|
||||||
|
branch = "master"
|
||||||
|
name = "github.com/alecthomas/units"
|
||||||
packages = ["."]
|
packages = ["."]
|
||||||
revision = "ea383cf3ba6ec950874b8486cd72356d007c768f"
|
revision = "2efee857e7cfd4f3d0138cc3cbb1b4966962b93a"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
@@ -19,23 +46,32 @@
|
|||||||
packages = ["."]
|
packages = ["."]
|
||||||
revision = "d9036e2120b5ddfa53f3ebccd618c4af275f47da"
|
revision = "d9036e2120b5ddfa53f3ebccd618c4af275f47da"
|
||||||
|
|
||||||
|
[[projects]]
|
||||||
|
name = "github.com/go-ini/ini"
|
||||||
|
packages = ["."]
|
||||||
|
revision = "ace140f73450505f33e8b8418216792275ae82a7"
|
||||||
|
version = "v1.35.0"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/go-ole/go-ole"
|
name = "github.com/go-ole/go-ole"
|
||||||
packages = [".","oleutil"]
|
packages = [
|
||||||
revision = "0e87ea779d9deb219633b828a023b32e1244dd57"
|
".",
|
||||||
version = "v1.2.0"
|
"oleutil"
|
||||||
|
]
|
||||||
|
revision = "a41e3c4b706f6ae8dfbff342b06e40fa4d2d0506"
|
||||||
|
version = "v1.2.1"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/golang/mock"
|
name = "github.com/golang/mock"
|
||||||
packages = ["gomock"]
|
packages = ["gomock"]
|
||||||
revision = "13f360950a79f5864a972c786a10a50e44b69541"
|
revision = "c34cdb4725f4c3844d095133c6e40e448b86589b"
|
||||||
version = "v1.0.0"
|
version = "v1.1.1"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "github.com/hashicorp/go-version"
|
name = "github.com/hashicorp/go-version"
|
||||||
packages = ["."]
|
packages = ["."]
|
||||||
revision = "4fe82ae3040f80a03d04d2cccb5606a626b8e1ee"
|
revision = "23480c0665776210b5fbbac6eaaee40e3e6a96b7"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
@@ -43,6 +79,12 @@
|
|||||||
packages = ["."]
|
packages = ["."]
|
||||||
revision = "bf9dde6d0d2c004a008c27aaee91170c786f6db8"
|
revision = "bf9dde6d0d2c004a008c27aaee91170c786f6db8"
|
||||||
|
|
||||||
|
[[projects]]
|
||||||
|
name = "github.com/mattn/go-shellwords"
|
||||||
|
packages = ["."]
|
||||||
|
revision = "02e3cf038dcea8290e44424da473dd12be796a8a"
|
||||||
|
version = "v1.0.3"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/montanaflynn/stats"
|
name = "github.com/montanaflynn/stats"
|
||||||
packages = ["."]
|
packages = ["."]
|
||||||
@@ -52,12 +94,25 @@
|
|||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "github.com/pborman/getopt"
|
name = "github.com/pborman/getopt"
|
||||||
packages = [".","v2"]
|
packages = [
|
||||||
|
".",
|
||||||
|
"v2"
|
||||||
|
]
|
||||||
revision = "7148bc3a4c3008adfcab60cbebfd0576018f330b"
|
revision = "7148bc3a4c3008adfcab60cbebfd0576018f330b"
|
||||||
|
|
||||||
|
[[projects]]
|
||||||
|
branch = "master"
|
||||||
|
name = "github.com/percona/go-mysql"
|
||||||
|
packages = ["query"]
|
||||||
|
revision = "82ed67a1d0f1779cd60a025c54e0827da0c0838b"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/percona/pmgo"
|
name = "github.com/percona/pmgo"
|
||||||
packages = [".","pmgomock"]
|
packages = [
|
||||||
|
".",
|
||||||
|
"pmgomock"
|
||||||
|
]
|
||||||
|
revision = "497d06e28f910fbe26d5d60f59d36284a6901c6f"
|
||||||
version = "0.5.2"
|
version = "0.5.2"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
@@ -69,14 +124,21 @@
|
|||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/satori/go.uuid"
|
name = "github.com/satori/go.uuid"
|
||||||
packages = ["."]
|
packages = ["."]
|
||||||
revision = "879c5887cd475cd7864858769793b2ceb0d44feb"
|
revision = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3"
|
||||||
version = "v1.1.0"
|
version = "v1.2.0"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
name = "github.com/shirou/gopsutil"
|
name = "github.com/shirou/gopsutil"
|
||||||
packages = ["cpu","host","internal/common","mem","net","process"]
|
packages = [
|
||||||
revision = "bfe3c2e8f406bf352bc8df81f98c752224867349"
|
"cpu",
|
||||||
version = "v2.17.11"
|
"host",
|
||||||
|
"internal/common",
|
||||||
|
"mem",
|
||||||
|
"net",
|
||||||
|
"process"
|
||||||
|
]
|
||||||
|
revision = "fc04d2dd9a512906a2604242b35275179e250eda"
|
||||||
|
version = "v2.18.03"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
@@ -94,30 +156,40 @@
|
|||||||
branch = "master"
|
branch = "master"
|
||||||
name = "go4.org"
|
name = "go4.org"
|
||||||
packages = ["reflectutil"]
|
packages = ["reflectutil"]
|
||||||
revision = "034d17a462f7b2dcd1a4a73553ec5357ff6e6c6e"
|
revision = "9599cf28b011184741f249bd9f9330756b506cbc"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "golang.org/x/crypto"
|
name = "golang.org/x/crypto"
|
||||||
packages = ["ssh/terminal"]
|
packages = ["ssh/terminal"]
|
||||||
revision = "94eea52f7b742c7cbe0b03b22f0c4c8631ece122"
|
revision = "d6449816ce06963d9d136eee5a56fca5b0616e7e"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "golang.org/x/net"
|
name = "golang.org/x/net"
|
||||||
packages = ["context"]
|
packages = ["context"]
|
||||||
revision = "d866cfc389cec985d6fda2859936a575a55a3ab6"
|
revision = "d41e8174641f662c5a2d1c7a5f9e828788eb8706"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "master"
|
branch = "master"
|
||||||
name = "golang.org/x/sys"
|
name = "golang.org/x/sys"
|
||||||
packages = ["unix","windows"]
|
packages = [
|
||||||
revision = "9418b3fdda89831e55f1d80702d6bf2f06d5438f"
|
"unix",
|
||||||
|
"windows"
|
||||||
|
]
|
||||||
|
revision = "b126b21c05a91c856b027c16779c12e3bf236954"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
branch = "v2"
|
branch = "v2"
|
||||||
name = "gopkg.in/mgo.v2"
|
name = "gopkg.in/mgo.v2"
|
||||||
packages = [".","bson","dbtest","internal/json","internal/sasl","internal/scram"]
|
packages = [
|
||||||
|
".",
|
||||||
|
"bson",
|
||||||
|
"dbtest",
|
||||||
|
"internal/json",
|
||||||
|
"internal/sasl",
|
||||||
|
"internal/scram"
|
||||||
|
]
|
||||||
revision = "3f83fa5005286a7fe593b055f0d7771a7dce4655"
|
revision = "3f83fa5005286a7fe593b055f0d7771a7dce4655"
|
||||||
|
|
||||||
[[projects]]
|
[[projects]]
|
||||||
@@ -129,6 +201,6 @@
|
|||||||
[solve-meta]
|
[solve-meta]
|
||||||
analyzer-name = "dep"
|
analyzer-name = "dep"
|
||||||
analyzer-version = 1
|
analyzer-version = 1
|
||||||
inputs-digest = "22594a7df9824c0a88d1de53c578d040b031dabd75d1fa852f7f775f4a71ae59"
|
inputs-digest = "fbf4dee6023d7297ebed697186e5687b1234ea1f49a100c54fa486c94306e5b9"
|
||||||
solver-name = "gps-cdcl"
|
solver-name = "gps-cdcl"
|
||||||
solver-version = 1
|
solver-version = 1
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
GO := go
|
GO := go
|
||||||
pkgs = $(shell find . -type d -name "pt-mongodb*" -exec basename {} \;)
|
pkgs = $(shell find . -type d -name "pt-*" -exec basename {} \;)
|
||||||
VERSION="3.0.8"
|
VERSION="3.0.8"
|
||||||
BUILD=$(shell date +%FT%T%z)
|
BUILD=$(shell date +%FT%T%z)
|
||||||
GOVERSION=$(shell go version | cut --delimiter=" " -f3)
|
GOVERSION=$(shell go version | cut --delimiter=" " -f3)
|
||||||
|
|||||||
95
src/go/pt-secure-collect/Makefile
Normal file
95
src/go/pt-secure-collect/Makefile
Normal file
@@ -0,0 +1,95 @@
|
|||||||
|
GO := go
|
||||||
|
pkgs = $(shell basename `git rev-parse --show-toplevel`)
|
||||||
|
VERSION ?=$(shell git describe --abbrev=0)
|
||||||
|
BUILD ?=$(shell date +%FT%T%z)
|
||||||
|
GOVERSION ?=$(shell go version | cut --delimiter=" " -f3)
|
||||||
|
COMMIT ?=$(shell git rev-parse HEAD)
|
||||||
|
BRANCH ?=$(shell git rev-parse --abbrev-ref HEAD)
|
||||||
|
GOPATH ?=${HOME}/go
|
||||||
|
|
||||||
|
MAKE_TARS = ''
|
||||||
|
CUR_DIR=$(shell pwd)
|
||||||
|
BIN_DIR=${CUR_DIR}/build
|
||||||
|
SOURCES=collect encryptor pt-secure-data
|
||||||
|
LDFLAGS="-X main.Version=${VERSION} -X main.Build=${BUILD} -X main.Commit=${COMMIT} -X main.Branch=${BRANCH} -X main.GoVersion=${GOVERSION} -s -w"
|
||||||
|
|
||||||
|
ifeq (${GOPATH},)
|
||||||
|
$(error GOPATH is not set)
|
||||||
|
endif
|
||||||
|
|
||||||
|
ifeq (,$(wildcard ${GOPATH}/src))
|
||||||
|
$(error Invalid GOPATH. There is no src dir in the GOPATH)
|
||||||
|
endif
|
||||||
|
|
||||||
|
ifeq ($(findstring ${GOPATH},${CUR_DIR}), )
|
||||||
|
$(error Wrong directorry for the project. It must be in $GOPATH/github/Percona-Lab/pt-secure-data)
|
||||||
|
endif
|
||||||
|
|
||||||
|
$(info )
|
||||||
|
$(info GOPATH..........: ${GOPATH})
|
||||||
|
$(info Build directory.: ${BIN_DIR})
|
||||||
|
$(info )
|
||||||
|
|
||||||
|
.PHONY: all style format build test vet tarball linux-amd64
|
||||||
|
|
||||||
|
default: prepare
|
||||||
|
@$(info Cleaning old tar files in ${BIN_DIR})
|
||||||
|
@rm -f ${BIN_DIR}/collect_*.tar.gz
|
||||||
|
@echo
|
||||||
|
@$(info Building in ${BIN_DIR})
|
||||||
|
@go build -ldflags ${LDFLAGS} -o ${BIN_DIR}/pt-secure-data *.go
|
||||||
|
|
||||||
|
prepare:
|
||||||
|
@$(info Checking if ${BIN_DIR} exists)
|
||||||
|
@mkdir -p ${BIN_DIR}
|
||||||
|
|
||||||
|
all: clean darwin-amd64-tar linux-amd64-tar windows-amd64-tar
|
||||||
|
|
||||||
|
clean: prepare
|
||||||
|
@$(info Cleaning binaries and tar.gz files in dir ${BIN_DIR})
|
||||||
|
@rm -f ${BIN_DIR}/pt-secure-data
|
||||||
|
@rm -f ${BIN_DIR}/pt-secure-data.exe
|
||||||
|
@rm -f ${BIN_DIR}/pt-secure-data_*.tar.gz
|
||||||
|
|
||||||
|
linux-amd64: prepare
|
||||||
|
@echo "Building linux/amd64 binaries in ${BIN_DIR}"
|
||||||
|
@GOOS=linux GOARCH=amd64 go build -ldflags ${LDFLAGS} -o ${BIN_DIR}/pt-secure-data *.go
|
||||||
|
|
||||||
|
linux-amd64-tar: linux-amd64
|
||||||
|
@tar cvzf ${BIN_DIR}/pt-secure-data_linux_amd64.tar.gz -C ${BIN_DIR} pt-secure-data
|
||||||
|
|
||||||
|
darwin-amd64:
|
||||||
|
@echo "Building darwin/amd64 binaries in ${BIN_DIR}"
|
||||||
|
@mkdir -p ${BIN_DIR}
|
||||||
|
@GOOS=darwin GOARCH=amd64 go build -ldflags ${LDFLAGS} -o ${BIN_DIR}/pt-secure-data *.go
|
||||||
|
|
||||||
|
darwin-amd64-tar: darwin-amd64
|
||||||
|
@tar cvzf ${BIN_DIR}/pt-secure-data_darwin_amd64.tar.gz -C ${BIN_DIR} pt-secure-data
|
||||||
|
|
||||||
|
windows-amd64: prepare
|
||||||
|
@echo "Building windows/amd64 binaries in ${BIN_DIR}"
|
||||||
|
@GOOS=windows GOARCH=amd64 go build -ldflags ${LDFLAGS} -o ${BIN_DIR}/pt-secure-data.exe *.go
|
||||||
|
|
||||||
|
windows-amd64-tar: windows-amd64
|
||||||
|
@tar cvzf ${BIN_DIR}/pt-secure-data_windows_amd64.tar.gz -C ${BIN_DIR} pt-secure-data.exe
|
||||||
|
|
||||||
|
style:
|
||||||
|
@echo ">> checking code style"
|
||||||
|
@! gofmt -d $(shell find . -path ./vendor -prune -o -name '*.go' -print) | grep '^'
|
||||||
|
|
||||||
|
test:
|
||||||
|
@echo ">> running tests"
|
||||||
|
@./runtests.sh
|
||||||
|
|
||||||
|
clean-tests:
|
||||||
|
@$(info Cleaning up docker containers used for tests)
|
||||||
|
@docker-compose down
|
||||||
|
|
||||||
|
format:
|
||||||
|
@echo ">> formatting code"
|
||||||
|
@$(GO) fmt $(pkgs)
|
||||||
|
|
||||||
|
vet:
|
||||||
|
@echo ">> vetting code"
|
||||||
|
@$(GO) vet $(pkgs)
|
||||||
|
|
||||||
88
src/go/pt-secure-collect/README.md
Normal file
88
src/go/pt-secure-collect/README.md
Normal file
@@ -0,0 +1,88 @@
|
|||||||
|
# pt-secure-data
|
||||||
|
Collect, sanitize, pack and encrypt data. By default, this program will collect the output of:
|
||||||
|
|
||||||
|
- `pt-stalk --no-stalk --iterations=2 --sleep=30 --host=$mysql-host --dest=$temp-dir --port=$mysql-port --user=$mysql-user --password=$mysql-pass`
|
||||||
|
- `pt-summary`
|
||||||
|
- `pt-mysql-summary --host=$mysql-host --port=$mysql-port --user=$mysql-user --password=$mysql-pass`
|
||||||
|
|
||||||
|
Internal variables placeholders will be replaced with the corresponding flag values. For example, `$mysql-host` will be replaced with the values specified in the `--mysql-host` flag.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
```
|
||||||
|
pt-secure-data [<flags>] <command> [<args> ...]
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
### Global flags
|
||||||
|
|Flag|Description|
|
||||||
|
|-----|-----|
|
||||||
|
|--help|Show context-sensitive help (also try --help-long and --help-man).|
|
||||||
|
|--debug|Enable debug log level.|
|
||||||
|
|
||||||
|
### **Commands**
|
||||||
|
#### **Help command**
|
||||||
|
Show help
|
||||||
|
|
||||||
|
#### **Collect command**
|
||||||
|
Collect, sanitize, pack and encrypt data from pt-tools.
|
||||||
|
Usage:
|
||||||
|
```
|
||||||
|
pt-secure-collect collect <flags>
|
||||||
|
```
|
||||||
|
|
||||||
|
|Flag|Description|
|
||||||
|
|-----|-----|
|
||||||
|
|--bin-dir|Directory having the Percona Toolkit binaries (if they are not in PATH).|
|
||||||
|
|--temp-dir|Temporary directory used for the data collection. Default: ${HOME}/data_collection\_{timestamp}|
|
||||||
|
|--include-dir|Include this dir into the sanitized tar file|
|
||||||
|
|--config-file|Path to the config file. Default: `~/.my.cnf`|
|
||||||
|
|--mysql-host|MySQL host. Default: `127.0.0.1`|
|
||||||
|
|--mysql-port|MySQL port. Default: `3306`|
|
||||||
|
|--mysql-user|MySQL user name.|
|
||||||
|
|--mysql-password|MySQL password.|
|
||||||
|
|--ask-mysql-pass|Ask MySQL password.|
|
||||||
|
|--extra-cmd|Also run this command as part of the data collection. This parameter can be used more than once.|
|
||||||
|
|--encrypt-password|Encrypt the output file using this password.<br>If ommited, it will be asked in the command line.|
|
||||||
|
|--no-collect|Do not collect data|
|
||||||
|
|--no-sanitize|Do not sanitize data|
|
||||||
|
|--no-encrypt|Do not encrypt the output file.|
|
||||||
|
|--no-sanitize-hostnames|Do not sanitize host names.|
|
||||||
|
|--no-sanitize-queries|Do not replace queries by their fingerprints.|
|
||||||
|
|--no-remove-temp-files|Do not remove temporary files.|
|
||||||
|
|
||||||
|
#### **Decrypt command**
|
||||||
|
Decrypt an encrypted file. The password will be requested from the terminal.
|
||||||
|
Usage:
|
||||||
|
```
|
||||||
|
pt-secure-collect decrypt [flags] <input file>
|
||||||
|
```
|
||||||
|
|Flag|Description|
|
||||||
|
|-----|------|
|
||||||
|
|--outfile|Write the output to this file.<br>If ommited, the output file name will be the same as the input file, adding the `.aes` extension|
|
||||||
|
|
||||||
|
|
||||||
|
#### **Encrypt command**
|
||||||
|
Encrypt a file. The password will be requested from the terminal.
|
||||||
|
Usage:
|
||||||
|
```
|
||||||
|
pt-secure-collect encrypt [flags] <input file>
|
||||||
|
```
|
||||||
|
|Flag|Description|
|
||||||
|
|-----|------|
|
||||||
|
|--outfile|Write the output to this file.<br>If ommited, the output file name will be the same as the input file, without the `.aes` extension|
|
||||||
|
|
||||||
|
|
||||||
|
#### **Sanitize command**
|
||||||
|
Replace queries in a file by their fingerprints and obfuscate hostnames.
|
||||||
|
Usage:
|
||||||
|
```
|
||||||
|
pt-secure-collect sanitize [flags]
|
||||||
|
```
|
||||||
|
|
||||||
|
|Flag|Description|
|
||||||
|
|-----|-----|
|
||||||
|
|--input-file| Input file. If not specified, the input will be Stdin.|
|
||||||
|
|--output-file|Output file. If not specified, the input will be Stdout.|
|
||||||
|
|--no-sanitize-hostnames|Do not sanitize host names.|
|
||||||
|
|--no-sanitize-queries|Do not replace queries by their fingerprints.|
|
||||||
|
|
||||||
251
src/go/pt-secure-collect/collect.go
Normal file
251
src/go/pt-secure-collect/collect.go
Normal file
@@ -0,0 +1,251 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"compress/gzip"
|
||||||
|
"crypto/sha256"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"os/user"
|
||||||
|
"path"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
log "github.com/Sirupsen/logrus"
|
||||||
|
shellwords "github.com/mattn/go-shellwords"
|
||||||
|
"github.com/percona/percona-toolkit/src/go/pt-secure-collect/sanitize"
|
||||||
|
"github.com/percona/percona-toolkit/src/go/pt-secure-collect/sanitize/util"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
)
|
||||||
|
|
||||||
|
func collectData(opts *cliOptions) error {
|
||||||
|
log.Infof("Temp directory is %q", *opts.TempDir)
|
||||||
|
|
||||||
|
if !*opts.NoCollect {
|
||||||
|
cmds, safeCmds, err := getCommandsToRun(defaultCmds, opts)
|
||||||
|
// Run the commands
|
||||||
|
if err = runCommands(cmds, safeCmds, *opts.TempDir); err != nil {
|
||||||
|
return errors.Wrap(err, "Cannot run data collection commands")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !*opts.NoSanitize {
|
||||||
|
log.Infof("Sanitizing output collected data")
|
||||||
|
err := processFiles(*opts.TempDir, *opts.IncludeDirs, *opts.TempDir, !*opts.NoSanitizeHostnames, !*opts.NoSanitizeQueries)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot sanitize files in %q", *opts.TempDir)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tarFile := fmt.Sprintf(path.Join(*opts.TempDir, path.Base(*opts.TempDir)+".tar.gz"))
|
||||||
|
log.Infof("Creating tar file %q", tarFile)
|
||||||
|
if err := tarit(tarFile, []string{*opts.TempDir}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !*opts.NoEncrypt && *opts.EncryptPassword != "" {
|
||||||
|
password := sha256.Sum256([]byte(*opts.EncryptPassword))
|
||||||
|
encryptedFile := tarFile + ".aes"
|
||||||
|
log.Infof("Encrypting %q file into %q", tarFile, encryptedFile)
|
||||||
|
encrypt(tarFile, encryptedFile, password)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func processFiles(dataDir string, includeDirs []string, outputDir string, sanitizeHostnames, sanitizeQueries bool) error {
|
||||||
|
dirs := []string{dataDir}
|
||||||
|
dirs = append(dirs, includeDirs...)
|
||||||
|
|
||||||
|
for _, dir := range dirs {
|
||||||
|
files, err := ioutil.ReadDir(dir)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot get the listing of %q", dir)
|
||||||
|
}
|
||||||
|
if len(files) == 0 {
|
||||||
|
return errors.Errorf("There are no files to sanitize in %q", dir)
|
||||||
|
}
|
||||||
|
log.Debug("Sanitization process start")
|
||||||
|
|
||||||
|
for _, file := range files {
|
||||||
|
if file.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
inputFile := path.Join(dir, file.Name())
|
||||||
|
log.Debugf("Reading %q", inputFile)
|
||||||
|
fh, err := os.Open(inputFile)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot open %q for reading", inputFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
lines, err := util.ReadLinesFromFile(fh)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot sanitize %q", inputFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Debugf("Sanitizing %q", inputFile)
|
||||||
|
sanitized := sanitize.Sanitize(lines, sanitizeHostnames, sanitizeQueries)
|
||||||
|
|
||||||
|
outfile := path.Join(outputDir, file.Name())
|
||||||
|
log.Debugf("Writing sanitized file to %q", outfile)
|
||||||
|
ofh, err := os.Create(outfile)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot open %q for writing", outfile)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = util.WriteLinesToFile(ofh, sanitized); err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot write sanitized file %q", outfile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func tarit(outfile string, srcPaths []string) error {
|
||||||
|
file, err := os.Create(outfile)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot create tar file %q", outfile)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
gw := gzip.NewWriter(file)
|
||||||
|
defer gw.Close()
|
||||||
|
|
||||||
|
tw := tar.NewWriter(gw)
|
||||||
|
defer tw.Close()
|
||||||
|
|
||||||
|
for _, srcPath := range srcPaths {
|
||||||
|
files, err := ioutil.ReadDir(srcPath)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot get the listing of %q", srcPath)
|
||||||
|
}
|
||||||
|
for _, file := range files {
|
||||||
|
// Ignore tar.gz files from previous runs
|
||||||
|
if strings.HasSuffix(file.Name(), ".tar.gz") {
|
||||||
|
log.Debugf("Skipping file %q", file.Name())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
log.Debugf("Adding %q to the tar file", file.Name())
|
||||||
|
if err := addFile(tw, srcPath, file); err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot add %q to the tar file %q", file.Name(), outfile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func getCommandsToRun(defaultCmds []string, opts *cliOptions) ([]*exec.Cmd, []string, error) {
|
||||||
|
log.Debug("Default commands to run:")
|
||||||
|
for i, cmd := range defaultCmds {
|
||||||
|
log.Debugf("%02d) %s", i, cmd)
|
||||||
|
}
|
||||||
|
cmdList := []string{}
|
||||||
|
cmds := []*exec.Cmd{}
|
||||||
|
safeCmds := []string{}
|
||||||
|
notAllowedCmdsRe := regexp.MustCompile("(rm|fdisk|rmdir)")
|
||||||
|
|
||||||
|
if !*opts.NoCollect {
|
||||||
|
cmdList = append(cmdList, defaultCmds...)
|
||||||
|
}
|
||||||
|
|
||||||
|
if *opts.AdditionalCmds != nil {
|
||||||
|
cmdList = append(cmdList, *opts.AdditionalCmds...)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, cmdstr := range cmdList {
|
||||||
|
cmdstr = strings.Replace(cmdstr, "$mysql-host", *opts.MySQLHost, -1)
|
||||||
|
cmdstr = strings.Replace(cmdstr, "$mysql-port", fmt.Sprintf("%d", *opts.MySQLPort), -1)
|
||||||
|
cmdstr = strings.Replace(cmdstr, "$mysql-user", *opts.MySQLUser, -1)
|
||||||
|
cmdstr = strings.Replace(cmdstr, "$temp-dir", *opts.TempDir, -1)
|
||||||
|
safeCmd := cmdstr
|
||||||
|
safeCmd = strings.Replace(safeCmd, "$mysql-pass", "********", -1)
|
||||||
|
cmdstr = strings.Replace(cmdstr, "$mysql-pass", *opts.MySQLPass, -1)
|
||||||
|
|
||||||
|
args, err := shellwords.Parse(cmdstr)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, errors.Wrapf(err, "Cannot parse %q", cmdstr)
|
||||||
|
}
|
||||||
|
if found := notAllowedCmdsRe.FindAllString(args[0], -1); len(found) > 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := exec.Command(args[0], args[1:]...)
|
||||||
|
cmds = append(cmds, cmd)
|
||||||
|
safeCmds = append(safeCmds, safeCmd)
|
||||||
|
}
|
||||||
|
return cmds, safeCmds, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func runCommands(cmds []*exec.Cmd, safeCmds []string, dataDir string) error {
|
||||||
|
for i := range cmds {
|
||||||
|
cmd := cmds[i]
|
||||||
|
safeCmd := safeCmds[i]
|
||||||
|
|
||||||
|
logFile := path.Join(dataDir, fmt.Sprintf("%s_%s.out", path.Base(cmd.Args[0]), time.Now().Format("2006-01-02_15_04_05")))
|
||||||
|
log.Infof("Creating output file %q", logFile)
|
||||||
|
fh, err := os.Create(logFile)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot create output file %s", logFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Infof("Running %s", safeCmd)
|
||||||
|
stdoutStderr, err := cmd.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
fh.WriteString(fmt.Sprintf("There was a problem running %s\n%s", safeCmd, err))
|
||||||
|
fh.Write(stdoutStderr)
|
||||||
|
fh.Close()
|
||||||
|
return errors.Wrapf(err, "\nThere was a problem running %s\n%s",
|
||||||
|
safeCmd, fmt.Sprintf("See %s for more details.", logFile))
|
||||||
|
}
|
||||||
|
fh.Write(stdoutStderr)
|
||||||
|
fh.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func addFile(tw *tar.Writer, srcPath string, fileInfo os.FileInfo) error {
|
||||||
|
file, err := os.Open(path.Join(srcPath, fileInfo.Name()))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
if _, err := file.Stat(); err == nil {
|
||||||
|
header, err := tar.FileInfoHeader(fileInfo, "")
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot create tar file header for %q", fileInfo.Name())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add the path since fileInfo.Name() only has the file name without the path
|
||||||
|
header.Name = path.Join(path.Base(srcPath), fileInfo.Name())
|
||||||
|
|
||||||
|
if err := tw.WriteHeader(header); err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot write file header for %q into the tar file", fileInfo.Name())
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := io.Copy(tw, file); err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot write file %q to the tar file", fileInfo.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func getTempDir() (string, error) {
|
||||||
|
user, err := user.Current()
|
||||||
|
if err != nil {
|
||||||
|
return "", errors.Wrap(err, "Cannot get current user information")
|
||||||
|
}
|
||||||
|
|
||||||
|
dir, err := ioutil.TempDir(user.HomeDir, "sanitize_")
|
||||||
|
if err != nil {
|
||||||
|
return "", errors.Wrap(err, "Cannot create temporary directory")
|
||||||
|
}
|
||||||
|
|
||||||
|
return dir, nil
|
||||||
|
}
|
||||||
35
src/go/pt-secure-collect/docker-compose.yml
Normal file
35
src/go/pt-secure-collect/docker-compose.yml
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
version: '3'
|
||||||
|
services:
|
||||||
|
mysql5.6:
|
||||||
|
image: ${MYSQL_IMAGE:-mysql:5.6}
|
||||||
|
ports:
|
||||||
|
- ${MYSQL_HOST:-127.0.0.1}:${MYSQL_PORT:-3306}:3306
|
||||||
|
environment:
|
||||||
|
- MYSQL_ALLOW_EMPTY_PASSWORD=yes
|
||||||
|
# MariaDB >= 10.0.12 doesn't enable Performance Schema by default so we need to do it manually
|
||||||
|
# https://mariadb.com/kb/en/mariadb/performance-schema-overview/#activating-the-performance-schema
|
||||||
|
command: --performance-schema --secure-file-priv=""
|
||||||
|
volumes:
|
||||||
|
- ./testdata/schema/:/docker-entrypoint-initdb.d/:rw
|
||||||
|
mysql5.7:
|
||||||
|
image: ${MYSQL_IMAGE:-mysql:5.7}
|
||||||
|
ports:
|
||||||
|
- ${MYSQL_HOST:-127.0.0.1}:${MYSQL_PORT:-3307}:3306
|
||||||
|
environment:
|
||||||
|
- MYSQL_ALLOW_EMPTY_PASSWORD=yes
|
||||||
|
# MariaDB >= 10.0.12 doesn't enable Performance Schema by default so we need to do it manually
|
||||||
|
# https://mariadb.com/kb/en/mariadb/performance-schema-overview/#activating-the-performance-schema
|
||||||
|
command: --performance-schema --secure-file-priv=""
|
||||||
|
volumes:
|
||||||
|
- ./testdata/schema/:/docker-entrypoint-initdb.d/:rw
|
||||||
|
mysql8.0:
|
||||||
|
image: ${MYSQL_IMAGE:-mysql:8.0.4}
|
||||||
|
ports:
|
||||||
|
- ${MYSQL_HOST:-127.0.0.1}:${MYSQL_PORT:-3308}:3306
|
||||||
|
environment:
|
||||||
|
- MYSQL_ALLOW_EMPTY_PASSWORD=yes
|
||||||
|
# MariaDB >= 10.0.12 doesn't enable Performance Schema by default so we need to do it manually
|
||||||
|
# https://mariadb.com/kb/en/mariadb/performance-schema-overview/#activating-the-performance-schema
|
||||||
|
command: --performance-schema --secure-file-priv=""
|
||||||
|
volumes:
|
||||||
|
- ./testdata/schema/:/docker-entrypoint-initdb.d/:rw
|
||||||
96
src/go/pt-secure-collect/encrypt.go
Normal file
96
src/go/pt-secure-collect/encrypt.go
Normal file
@@ -0,0 +1,96 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/aes"
|
||||||
|
"crypto/cipher"
|
||||||
|
"crypto/sha256"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
log "github.com/Sirupsen/logrus"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
)
|
||||||
|
|
||||||
|
func encryptorCmd(opts *cliOptions) (err error) {
|
||||||
|
password := sha256.Sum256([]byte(*opts.EncryptPassword))
|
||||||
|
|
||||||
|
switch opts.Command {
|
||||||
|
case "decrypt":
|
||||||
|
if *opts.DecryptOutFile == "" && strings.HasSuffix(*opts.DecryptInFile, ".aes") {
|
||||||
|
*opts.DecryptOutFile = strings.TrimSuffix(filepath.Base(*opts.DecryptInFile), ".aes")
|
||||||
|
}
|
||||||
|
log.Infof("Decrypting file %q into %q", *opts.DecryptInFile, *opts.DecryptOutFile)
|
||||||
|
err = decrypt(*opts.DecryptInFile, *opts.DecryptOutFile, password)
|
||||||
|
case "encrypt":
|
||||||
|
if *opts.EncryptOutFile == "" {
|
||||||
|
*opts.EncryptOutFile = filepath.Base(*opts.EncryptInFile) + ".aes"
|
||||||
|
}
|
||||||
|
log.Infof("Encrypting file %q into %q", *opts.EncryptInFile, *opts.EncryptOutFile)
|
||||||
|
err = encrypt(*opts.EncryptInFile, *opts.EncryptOutFile, password)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func encrypt(infile, outfile string, pass [32]byte) error {
|
||||||
|
key := pass[:]
|
||||||
|
inFile, err := os.Open(infile)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot open input file %q", infile)
|
||||||
|
}
|
||||||
|
defer inFile.Close()
|
||||||
|
|
||||||
|
block, err := aes.NewCipher(key)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot create a new cipher")
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the key is unique for each ciphertext, then it's ok to use a zero IV.
|
||||||
|
var iv [aes.BlockSize]byte
|
||||||
|
stream := cipher.NewOFB(block, iv[:])
|
||||||
|
|
||||||
|
outFile, err := os.OpenFile(outfile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot create output file %q", outfile)
|
||||||
|
}
|
||||||
|
defer outFile.Close()
|
||||||
|
|
||||||
|
writer := &cipher.StreamWriter{S: stream, W: outFile}
|
||||||
|
// Copy the input file to the output file, encrypting as we go.
|
||||||
|
if _, err := io.Copy(writer, inFile); err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot write to output file %q", outfile)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func decrypt(infile, outfile string, pass [32]byte) error {
|
||||||
|
key := pass[:]
|
||||||
|
inFile, err := os.Open(infile)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot open %q for reading", infile)
|
||||||
|
}
|
||||||
|
defer inFile.Close()
|
||||||
|
|
||||||
|
block, err := aes.NewCipher(key)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrap(err, "Cannot create the cipher")
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the key is unique for each ciphertext, then it's ok to use a zero IV.
|
||||||
|
var iv [aes.BlockSize]byte
|
||||||
|
stream := cipher.NewOFB(block, iv[:])
|
||||||
|
|
||||||
|
outFile, err := os.OpenFile(outfile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot open %q for writing", outfile)
|
||||||
|
}
|
||||||
|
defer outFile.Close()
|
||||||
|
|
||||||
|
reader := &cipher.StreamReader{S: stream, R: inFile}
|
||||||
|
// Copy the input file to the output file, decrypting as we go.
|
||||||
|
if _, err := io.Copy(outFile, reader); err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot write to output file %q", outfile)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
387
src/go/pt-secure-collect/main.go
Normal file
387
src/go/pt-secure-collect/main.go
Normal file
@@ -0,0 +1,387 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"os/user"
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/Sirupsen/logrus"
|
||||||
|
log "github.com/Sirupsen/logrus"
|
||||||
|
"github.com/alecthomas/kingpin"
|
||||||
|
"github.com/go-ini/ini"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"golang.org/x/crypto/ssh/terminal"
|
||||||
|
)
|
||||||
|
|
||||||
|
type cliOptions struct {
|
||||||
|
Command string
|
||||||
|
Debug *bool
|
||||||
|
|
||||||
|
DecryptCommand *kingpin.CmdClause
|
||||||
|
DecryptInFile *string
|
||||||
|
DecryptOutFile *string
|
||||||
|
|
||||||
|
EncryptCommand *kingpin.CmdClause
|
||||||
|
EncryptInFile *string
|
||||||
|
EncryptOutFile *string
|
||||||
|
|
||||||
|
CollectCommand *kingpin.CmdClause
|
||||||
|
BinDir *string
|
||||||
|
TempDir *string // in case Percona Toolkit is not in the PATH
|
||||||
|
IncludeDirs *[]string
|
||||||
|
ConfigFile *string // .my.cnf file
|
||||||
|
EncryptPassword *string // if set, it will produce an encrypted .aes file
|
||||||
|
AdditionalCmds *[]string
|
||||||
|
AskMySQLPass *bool
|
||||||
|
MySQLHost *string
|
||||||
|
MySQLPort *int
|
||||||
|
MySQLUser *string
|
||||||
|
MySQLPass *string
|
||||||
|
|
||||||
|
NoEncrypt *bool
|
||||||
|
NoSanitize *bool
|
||||||
|
NoSanitizeHostnames *bool
|
||||||
|
NoSanitizeQueries *bool
|
||||||
|
NoCollect *bool
|
||||||
|
NoRemoveTempFiles *bool
|
||||||
|
|
||||||
|
SanitizeCommand *kingpin.CmdClause
|
||||||
|
SanitizeInputFile *string
|
||||||
|
SanitizeOutputFile *string
|
||||||
|
DontSanitizeHostnames *bool
|
||||||
|
DontSanitizeQueries *bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type myDefaults struct {
|
||||||
|
MySQLHost string
|
||||||
|
MySQLPort int
|
||||||
|
MySQLUser string
|
||||||
|
MySQLPass string
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
DecryptCmd = "decrypt"
|
||||||
|
EncryptCmd = "encrypt"
|
||||||
|
CollectCmd = "collect"
|
||||||
|
SanitizeCmd = "sanitize"
|
||||||
|
DefaultMySQLHost = "127.0.0.1"
|
||||||
|
DefaultMySQLPort = 3306
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
defaultCmds = []string{
|
||||||
|
"pt-stalk --no-stalk --iterations=2 --sleep=30 --host=$mysql-host --dest=$temp-dir --port=$mysql-port --user=$mysql-user --password=$mysql-pass",
|
||||||
|
"pt-summary",
|
||||||
|
"pt-mysql-summary --host=$mysql-host --port=$mysql-port --user=$mysql-user --password=$mysql-pass",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
customFormatter := new(logrus.TextFormatter)
|
||||||
|
customFormatter.TimestampFormat = "2006-01-02 15:04:05"
|
||||||
|
logrus.SetFormatter(customFormatter)
|
||||||
|
customFormatter.FullTimestamp = true
|
||||||
|
|
||||||
|
u, err := user.Current()
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Cannot get current user: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
opts, err := processCliParams(u.HomeDir, nil)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch opts.Command {
|
||||||
|
case CollectCmd:
|
||||||
|
if _, err = os.Stat(*opts.TempDir); os.IsNotExist(err) {
|
||||||
|
log.Infof("Creating temporary directory: %s", *opts.TempDir)
|
||||||
|
if err = os.Mkdir(*opts.TempDir, os.ModePerm); err != nil {
|
||||||
|
log.Fatalf("Cannot create temporary dirextory %q: %s", *opts.TempDir, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
err = collectData(opts)
|
||||||
|
if err != nil && !*opts.NoRemoveTempFiles {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
if !*opts.NoRemoveTempFiles {
|
||||||
|
if err = removeTempFiles(*opts.TempDir, !*opts.NoEncrypt); err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case EncryptCmd, DecryptCmd:
|
||||||
|
err = encryptorCmd(opts)
|
||||||
|
case SanitizeCmd:
|
||||||
|
err = sanitizeFile(opts)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func removeTempFiles(tempDir string, removeTarFile bool) error {
|
||||||
|
tarFile := path.Base(tempDir) + ".tar.gz"
|
||||||
|
encryptedFile := tarFile + ".aes"
|
||||||
|
files, err := ioutil.ReadDir(tempDir)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot get the listing of %q", tempDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, file := range files {
|
||||||
|
if file.Name() == encryptedFile {
|
||||||
|
log.Infof("Skipping encrypted file %q", encryptedFile)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if file.Name() == tarFile && !removeTarFile {
|
||||||
|
log.Infof("Skipping tar.gz file %q", tarFile)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
filename := path.Join(tempDir, file.Name())
|
||||||
|
log.Debugf("Removing file %q", filename)
|
||||||
|
if err = os.Remove(filename); err != nil {
|
||||||
|
log.Warnf("Cannot remove %q: %s", filename, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func processCliParams(baseTempPath string, usageWriter io.Writer) (*cliOptions, error) {
|
||||||
|
var err error
|
||||||
|
tmpdir := path.Join(baseTempPath, fmt.Sprintf("data_collection_%s", time.Now().Format("2006-01-02_15_04_05")))
|
||||||
|
|
||||||
|
// Do not remove the extra space after \n. That's to trick the help template to not to remove the new line
|
||||||
|
msg := "Collect, sanitize, pack and encrypt data.\nBy default, this program will collect the output of:"
|
||||||
|
for _, cmd := range defaultCmds {
|
||||||
|
msg += "\n " + cmd
|
||||||
|
}
|
||||||
|
msg += "\n "
|
||||||
|
|
||||||
|
app := kingpin.New("pt-secure-data", msg)
|
||||||
|
if usageWriter != nil {
|
||||||
|
app.UsageWriter(usageWriter)
|
||||||
|
app.Terminate(nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
opts := &cliOptions{
|
||||||
|
CollectCommand: app.Command(CollectCmd, "Collect, sanitize, pack and encrypt data from pt-tools."),
|
||||||
|
DecryptCommand: app.Command(DecryptCmd, "Decrypt an encrypted file. The password will be requested from the terminal."),
|
||||||
|
EncryptCommand: app.Command(EncryptCmd, "Encrypt a file. The password will be requested from the terminal."),
|
||||||
|
SanitizeCommand: app.Command(SanitizeCmd, "Replace queries in a file by their fingerprints and obfuscate hostnames."),
|
||||||
|
Debug: app.Flag("debug", "Enable debug log level.").Bool(),
|
||||||
|
}
|
||||||
|
// Decrypt command flags
|
||||||
|
opts.DecryptInFile = opts.DecryptCommand.Arg("infile", "Encrypted file.").Required().String()
|
||||||
|
opts.DecryptOutFile = opts.DecryptCommand.Flag("outfile", "Unencrypted file. Default: same name without .aes extension").String()
|
||||||
|
|
||||||
|
// Encrypt command flags
|
||||||
|
opts.EncryptInFile = opts.EncryptCommand.Arg("infile", "Unencrypted file.").Required().String()
|
||||||
|
opts.EncryptOutFile = opts.EncryptCommand.Flag("outfile", "Encrypted file. Default: <input file>.aes").String()
|
||||||
|
|
||||||
|
// Collect command flags
|
||||||
|
opts.BinDir = opts.CollectCommand.Flag("bin-dir", "Directory having the Percona Toolkit binaries (if they are not in PATH).").String()
|
||||||
|
opts.TempDir = opts.CollectCommand.Flag("temp-dir", "Temporary directory used for the data collection.").Default(tmpdir).String()
|
||||||
|
opts.IncludeDirs = opts.CollectCommand.Flag("include-dir", "Include this dir into the sanitized tar file").Strings()
|
||||||
|
// MySQL related flags
|
||||||
|
opts.ConfigFile = opts.CollectCommand.Flag("config-file", "Path to the config file.").Default("~/.my.cnf").String()
|
||||||
|
opts.MySQLHost = opts.CollectCommand.Flag("mysql-host", "MySQL host.").String()
|
||||||
|
opts.MySQLPort = opts.CollectCommand.Flag("mysql-port", "MySQL port.").Int()
|
||||||
|
opts.MySQLUser = opts.CollectCommand.Flag("mysql-user", "MySQL user name.").String()
|
||||||
|
opts.MySQLPass = opts.CollectCommand.Flag("mysql-password", "MySQL password.").String()
|
||||||
|
opts.AskMySQLPass = opts.CollectCommand.Flag("ask-mysql-pass", "Ask MySQL password.").Bool()
|
||||||
|
// Aditional flags
|
||||||
|
opts.AdditionalCmds = opts.CollectCommand.Flag("extra-cmd",
|
||||||
|
"Also run this command as part of the data collection. This parameter can be used more than once.").Strings()
|
||||||
|
opts.EncryptPassword = opts.CollectCommand.Flag("encrypt-password", "Encrypt the output file using this password."+
|
||||||
|
" If ommited, the file won't be encrypted.").String()
|
||||||
|
// No-Flags
|
||||||
|
opts.NoCollect = opts.CollectCommand.Flag("no-collect", "Do not collect data").Bool()
|
||||||
|
opts.NoSanitize = opts.CollectCommand.Flag("no-sanitize", "Sanitize data").Bool()
|
||||||
|
opts.NoEncrypt = opts.CollectCommand.Flag("no-encrypt", "Do not encrypt the output file.").Bool()
|
||||||
|
opts.NoSanitizeHostnames = opts.CollectCommand.Flag("no-sanitize-hostnames", "Don't sanitize host names.").Bool()
|
||||||
|
opts.NoSanitizeQueries = opts.CollectCommand.Flag("no-sanitize-queries", "Do not replace queries by their fingerprints.").Bool()
|
||||||
|
opts.NoRemoveTempFiles = opts.CollectCommand.Flag("no-remove-temp-files", "Do not remove temporary files.").Bool()
|
||||||
|
|
||||||
|
// Sanitize command flags
|
||||||
|
opts.SanitizeInputFile = opts.SanitizeCommand.Flag("input-file", "Input file. If not specified, the input will be Stdin.").String()
|
||||||
|
opts.SanitizeOutputFile = opts.SanitizeCommand.Flag("output-file", "Output file. If not specified, the input will be Stdout.").String()
|
||||||
|
opts.DontSanitizeHostnames = opts.SanitizeCommand.Flag("no-sanitize-hostnames", "Don't sanitize host names.").Bool()
|
||||||
|
opts.DontSanitizeQueries = opts.SanitizeCommand.Flag("no-sanitize-queries", "Don't replace queries by their fingerprints.").Bool()
|
||||||
|
|
||||||
|
opts.Command, err = app.Parse(os.Args[1:])
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if *opts.Debug {
|
||||||
|
log.SetLevel(log.DebugLevel)
|
||||||
|
}
|
||||||
|
|
||||||
|
*opts.BinDir = expandHomeDir(*opts.BinDir)
|
||||||
|
*opts.ConfigFile = expandHomeDir(*opts.ConfigFile)
|
||||||
|
*opts.TempDir = expandHomeDir(*opts.TempDir)
|
||||||
|
for _, incDir := range *opts.IncludeDirs {
|
||||||
|
incDir = expandHomeDir(incDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
if *opts.BinDir != "" {
|
||||||
|
os.Setenv("PATH", fmt.Sprintf("%s%s%s", *opts.BinDir, string(os.PathListSeparator), os.Getenv("PATH")))
|
||||||
|
}
|
||||||
|
|
||||||
|
lp, err := exec.LookPath("pt-summary")
|
||||||
|
if (err != nil || lp == "") && *opts.BinDir == "" && opts.Command == "collect" && !*opts.NoCollect {
|
||||||
|
return nil, errors.New("Cannot find Percona Toolkit binaries. Please run this tool again using --bin-dir parameter")
|
||||||
|
}
|
||||||
|
|
||||||
|
switch opts.Command {
|
||||||
|
case CollectCmd:
|
||||||
|
mycnf, err := getParamsFromMyCnf(*opts.ConfigFile)
|
||||||
|
if err == nil {
|
||||||
|
if err = validateMySQLParams(opts, mycnf); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if *opts.AskMySQLPass {
|
||||||
|
if err = askMySQLPassword(opts); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
err = askEncryptionPassword(opts, true)
|
||||||
|
case EncryptCmd:
|
||||||
|
err = askEncryptionPassword(opts, true)
|
||||||
|
case DecryptCmd:
|
||||||
|
if !strings.HasSuffix(*opts.DecryptInFile, ".aes") && *opts.DecryptOutFile == "" {
|
||||||
|
return nil, fmt.Errorf("Input file does not have .aes extension. I cannot infer the output file")
|
||||||
|
}
|
||||||
|
err = askEncryptionPassword(opts, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return opts, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateMySQLParams(opts *cliOptions, mycnf *myDefaults) error {
|
||||||
|
if *opts.MySQLPort == 0 && mycnf.MySQLPort > 0 {
|
||||||
|
log.Debugf("Setting default port from config file")
|
||||||
|
*opts.MySQLPort = mycnf.MySQLPort
|
||||||
|
}
|
||||||
|
if *opts.MySQLHost == "" && mycnf.MySQLHost != "" {
|
||||||
|
*opts.MySQLHost = mycnf.MySQLHost
|
||||||
|
log.Debugf("Setting default host from config file")
|
||||||
|
}
|
||||||
|
if *opts.MySQLUser == "" && mycnf.MySQLUser != "" {
|
||||||
|
log.Debugf("Setting default user from config file")
|
||||||
|
*opts.MySQLUser = mycnf.MySQLUser
|
||||||
|
}
|
||||||
|
if *opts.MySQLPass == "" && mycnf.MySQLPass != "" {
|
||||||
|
log.Debugf("Setting default password from config file")
|
||||||
|
*opts.MySQLPass = mycnf.MySQLPass
|
||||||
|
}
|
||||||
|
|
||||||
|
if *opts.MySQLHost == "" {
|
||||||
|
log.Debugf("MySQL host is empty. Setting it to %s", DefaultMySQLHost)
|
||||||
|
*opts.MySQLHost = DefaultMySQLHost
|
||||||
|
}
|
||||||
|
if *opts.MySQLPort == 0 {
|
||||||
|
log.Debugf("MySQL port is empty. Setting it to %d", DefaultMySQLPort)
|
||||||
|
*opts.MySQLPort = DefaultMySQLPort
|
||||||
|
}
|
||||||
|
if *opts.MySQLUser == "" {
|
||||||
|
return fmt.Errorf("MySQL user cannot be empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func askMySQLPassword(opts *cliOptions) error {
|
||||||
|
if *opts.AskMySQLPass {
|
||||||
|
fmt.Printf("MySQL password for user %q:", *opts.MySQLUser)
|
||||||
|
passb, err := terminal.ReadPassword(0)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrap(err, "Cannot read MySQL password from the terminal")
|
||||||
|
}
|
||||||
|
*opts.MySQLPass = string(passb)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func askEncryptionPassword(opts *cliOptions, requireConfirmation bool) error {
|
||||||
|
if !*opts.NoEncrypt && *opts.EncryptPassword == "" {
|
||||||
|
fmt.Print("Encryption password: ")
|
||||||
|
passa, err := terminal.ReadPassword(0)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrap(err, "Cannot read encryption password from the terminal")
|
||||||
|
}
|
||||||
|
fmt.Println("")
|
||||||
|
if requireConfirmation {
|
||||||
|
fmt.Print("Re type password: ")
|
||||||
|
passb, err := terminal.ReadPassword(0)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrap(err, "Cannot read encryption password confirmation from the terminal")
|
||||||
|
}
|
||||||
|
fmt.Println("")
|
||||||
|
if string(passa) != string(passb) {
|
||||||
|
return errors.New("Passwords don't match")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
*opts.EncryptPassword = string(passa)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func getParamsFromMyCnf(configFile string) (*myDefaults, error) {
|
||||||
|
log.Debugf("Reading default MySQL parameters from config file: %q", configFile)
|
||||||
|
if configFile == "" {
|
||||||
|
return nil, fmt.Errorf("Config file cannot be empty")
|
||||||
|
}
|
||||||
|
configFile = expandHomeDir(configFile)
|
||||||
|
|
||||||
|
cfg, err := ini.Load(configFile)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrapf(err, "Cannot read config from %q", configFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
sec, err := cfg.GetSection("client")
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrapf(err, "Cannot read [client] section from %q", configFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
mycnf := &myDefaults{}
|
||||||
|
|
||||||
|
if val, err := sec.GetKey("user"); err == nil {
|
||||||
|
mycnf.MySQLUser = val.String()
|
||||||
|
}
|
||||||
|
if val, err := sec.GetKey("password"); err == nil {
|
||||||
|
mycnf.MySQLPass = val.String()
|
||||||
|
}
|
||||||
|
if val, err := sec.GetKey("host"); err == nil {
|
||||||
|
mycnf.MySQLHost = val.String()
|
||||||
|
}
|
||||||
|
if val, err := sec.GetKey("port"); err == nil {
|
||||||
|
if mycnf.MySQLPort, err = val.Int(); err != nil {
|
||||||
|
return nil, errors.Wrapf(err, "Cannot parse %q as the port number", val.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.Debugf("mycnf: %+v\n", *mycnf)
|
||||||
|
return mycnf, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func expandHomeDir(path string) string {
|
||||||
|
usr, _ := user.Current()
|
||||||
|
dir := usr.HomeDir
|
||||||
|
|
||||||
|
if len(path) > 1 && path[:2] == "~/" {
|
||||||
|
path = filepath.Join(dir, path[2:])
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
42
src/go/pt-secure-collect/main_test.go
Normal file
42
src/go/pt-secure-collect/main_test.go
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestProcessCliParams(t *testing.T) {
|
||||||
|
var output bytes.Buffer
|
||||||
|
writer := bufio.NewWriter(&output)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
Args []string
|
||||||
|
WantOpts *cliOptions
|
||||||
|
WantErr bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
Args: []string{"pt-sanitize-data", "llll"},
|
||||||
|
WantOpts: nil,
|
||||||
|
WantErr: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, test := range tests {
|
||||||
|
os.Args = test.Args
|
||||||
|
opts, err := processCliParams(os.TempDir(), writer)
|
||||||
|
writer.Flush()
|
||||||
|
if test.WantErr && err == nil {
|
||||||
|
t.Errorf("Test #%d expected error, have nil", i)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(opts, test.WantOpts) {
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollect(t *testing.T) {
|
||||||
|
|
||||||
|
}
|
||||||
42
src/go/pt-secure-collect/sanitize.go
Normal file
42
src/go/pt-secure-collect/sanitize.go
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/percona/percona-toolkit/src/go/pt-secure-collect/sanitize"
|
||||||
|
"github.com/percona/percona-toolkit/src/go/pt-secure-collect/sanitize/util"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
)
|
||||||
|
|
||||||
|
func sanitizeFile(opts *cliOptions) error {
|
||||||
|
var err error
|
||||||
|
ifh := os.Stdin
|
||||||
|
ofh := os.Stdout
|
||||||
|
|
||||||
|
if *opts.SanitizeInputFile != "" {
|
||||||
|
ifh, err = os.Open(*opts.SanitizeInputFile)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot open %q for reading", *opts.SanitizeInputFile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if *opts.SanitizeOutputFile != "" {
|
||||||
|
ifh, err = os.Create(*opts.SanitizeOutputFile)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot create output file %q", *opts.SanitizeOutputFile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
lines, err := util.ReadLinesFromFile(ifh)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot read input file %q", *opts.SanitizeInputFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
sanitized := sanitize.Sanitize(lines, !*opts.DontSanitizeHostnames, !*opts.DontSanitizeQueries)
|
||||||
|
|
||||||
|
if err = util.WriteLinesToFile(ofh, sanitized); err != nil {
|
||||||
|
return errors.Wrapf(err, "Cannot write output file %q", *opts.SanitizeOutputFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
115
src/go/pt-secure-collect/sanitize/sanitize.go
Normal file
115
src/go/pt-secure-collect/sanitize/sanitize.go
Normal file
@@ -0,0 +1,115 @@
|
|||||||
|
package sanitize
|
||||||
|
|
||||||
|
import (
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/percona/go-mysql/query"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
hostnameRE = regexp.MustCompile(`(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)+([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-][A-Za-z0-9]){2,3}(?:\W)`)
|
||||||
|
queryLineRe []*regexp.Regexp
|
||||||
|
queryInLineRe []*regexp.Regexp
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
statements := []string{
|
||||||
|
"CREATE (TABLE|VIEW|DEFINER)",
|
||||||
|
"DROP (DATABASE|TABLE|VIEW|DEFINER)",
|
||||||
|
"INSERT INTO",
|
||||||
|
"REPLACE INTO",
|
||||||
|
"UPDATE",
|
||||||
|
"SELECT.*FROM.*",
|
||||||
|
"SET ",
|
||||||
|
"SHOW TABLES",
|
||||||
|
"SHOW DATABASES",
|
||||||
|
"COMMIT",
|
||||||
|
"LOAD DATA",
|
||||||
|
}
|
||||||
|
for _, re := range statements {
|
||||||
|
queryLineRe = append(queryLineRe, regexp.MustCompile("(?i)^"+re))
|
||||||
|
queryInLineRe = append(queryInLineRe, regexp.MustCompile(("(?im)(" + re + ".*)")))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Sanitize(lines []string, hostnames, queries bool) []string {
|
||||||
|
joined := joinQueryLines(lines)
|
||||||
|
if queries {
|
||||||
|
sanitizeQueries(joined)
|
||||||
|
}
|
||||||
|
if hostnames {
|
||||||
|
sanitizeHostnames(joined)
|
||||||
|
}
|
||||||
|
return joined
|
||||||
|
}
|
||||||
|
|
||||||
|
func sanitizeHostnames(lines []string) {
|
||||||
|
for i := range lines {
|
||||||
|
lines[i] = hostnameRE.ReplaceAllStringFunc(lines[i], replaceHostname)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func sanitizeQueries(lines []string) {
|
||||||
|
for i := range lines {
|
||||||
|
for _, re := range queryInLineRe {
|
||||||
|
lines[i] = re.ReplaceAllStringFunc(lines[i], queryToFingerprint)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func joinQueryLines(lines []string) []string {
|
||||||
|
inQuery := false
|
||||||
|
joined := []string{}
|
||||||
|
queryString := ""
|
||||||
|
|
||||||
|
separator := ""
|
||||||
|
|
||||||
|
for _, line := range lines {
|
||||||
|
if !inQuery && mightBeAQueryLine(line) {
|
||||||
|
inQuery = true
|
||||||
|
}
|
||||||
|
if inQuery {
|
||||||
|
if strings.HasPrefix(line, "***") {
|
||||||
|
joined = append(joined, queryString)
|
||||||
|
queryString = ""
|
||||||
|
joined = append(joined, line)
|
||||||
|
inQuery = false
|
||||||
|
separator = ""
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
queryString += separator + line
|
||||||
|
separator = "\n"
|
||||||
|
if !strings.HasSuffix(strings.TrimSpace(line), ";") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
inQuery = false
|
||||||
|
separator = ""
|
||||||
|
joined = append(joined, queryString)
|
||||||
|
queryString = ""
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
joined = append(joined, line)
|
||||||
|
}
|
||||||
|
return joined
|
||||||
|
}
|
||||||
|
|
||||||
|
func replaceHostname(s string) string {
|
||||||
|
if strings.HasSuffix(s, ":") {
|
||||||
|
return "<hostname>:"
|
||||||
|
}
|
||||||
|
return "hostname"
|
||||||
|
}
|
||||||
|
|
||||||
|
func queryToFingerprint(q string) string {
|
||||||
|
return query.Fingerprint(q)
|
||||||
|
}
|
||||||
|
|
||||||
|
func mightBeAQueryLine(query string) bool {
|
||||||
|
for _, re := range queryLineRe {
|
||||||
|
if re.MatchString(query) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
45
src/go/pt-secure-collect/sanitize/util/util.go
Normal file
45
src/go/pt-secure-collect/sanitize/util/util.go
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
package util
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"bytes"
|
||||||
|
"encoding/gob"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
)
|
||||||
|
|
||||||
|
func ReadLinesFromFile(fh *os.File) ([]string, error) {
|
||||||
|
lines := []string{}
|
||||||
|
reader := bufio.NewReader(fh)
|
||||||
|
|
||||||
|
line, err := reader.ReadString('\n')
|
||||||
|
for err == nil {
|
||||||
|
lines = append(lines, strings.TrimRight(line, "\n"))
|
||||||
|
line, err = reader.ReadString('\n')
|
||||||
|
}
|
||||||
|
return lines, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func WriteLinesToFile(ofh *os.File, lines []string) error {
|
||||||
|
for _, line := range lines {
|
||||||
|
if _, err := ofh.WriteString(line + "\n"); err != nil {
|
||||||
|
return errors.Wrap(err, "Cannot write output file")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func LinesToBytes(lines []string) []byte {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
gob.NewEncoder(buf).Encode(lines)
|
||||||
|
return buf.Bytes()
|
||||||
|
}
|
||||||
|
|
||||||
|
func BytesToLines(buf []byte) []string {
|
||||||
|
reader := bytes.NewReader(buf)
|
||||||
|
lines := []string{}
|
||||||
|
gob.NewDecoder(reader).Decode(&lines)
|
||||||
|
return lines
|
||||||
|
}
|
||||||
166
src/go/pt-secure-collect/testdata/2018_03_05_13_24_55-innodbstatus1
vendored
Normal file
166
src/go/pt-secure-collect/testdata/2018_03_05_13_24_55-innodbstatus1
vendored
Normal file
@@ -0,0 +1,166 @@
|
|||||||
|
*************************** 1. row ***************************
|
||||||
|
Type: InnoDB
|
||||||
|
Name:
|
||||||
|
Status:
|
||||||
|
=====================================
|
||||||
|
180305 13:24:56 INNODB MONITOR OUTPUT
|
||||||
|
=====================================
|
||||||
|
Per second averages calculated from the last 4 seconds
|
||||||
|
-----------------
|
||||||
|
BACKGROUND THREAD
|
||||||
|
-----------------
|
||||||
|
srv_master_thread loops: 10426619 1_second, 10426491 sleeps, 1039481 10_second, 114406 background, 114405 flush
|
||||||
|
srv_master_thread log flush and writes: 10573068
|
||||||
|
----------
|
||||||
|
SEMAPHORES
|
||||||
|
----------
|
||||||
|
OS WAIT ARRAY INFO: reservation count 36440343, signal count 977828096
|
||||||
|
Mutex spin waits 12033970052, rounds 5762849775, OS waits 2335696
|
||||||
|
RW-shared spins 263735499, rounds 1073398879, OS waits 17684023
|
||||||
|
RW-excl spins 31045741, rounds 1579081322, OS waits 15143736
|
||||||
|
Spin rounds per wait: 0.48 mutex, 4.07 RW-shared, 50.86 RW-excl
|
||||||
|
--------
|
||||||
|
FILE I/O
|
||||||
|
--------
|
||||||
|
I/O thread 0 state: waiting for completed aio requests (insert buffer thread)
|
||||||
|
I/O thread 1 state: waiting for completed aio requests (log thread)
|
||||||
|
I/O thread 2 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 3 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 4 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 5 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 6 state: waiting for completed aio requests (write thread)
|
||||||
|
I/O thread 7 state: waiting for completed aio requests (write thread)
|
||||||
|
I/O thread 8 state: waiting for completed aio requests (write thread)
|
||||||
|
I/O thread 9 state: waiting for completed aio requests (write thread)
|
||||||
|
Pending normal aio reads: 0 [0, 0, 0, 0] , aio writes: 0 [0, 0, 0, 0] ,
|
||||||
|
ibuf aio reads: 0, log i/o's: 0, sync i/o's: 0
|
||||||
|
Pending flushes (fsync) log: 0; buffer pool: 0
|
||||||
|
69640821 OS file reads, 648910440 OS file writes, 26513529 OS fsyncs
|
||||||
|
0.00 reads/s, 0 avg bytes/read, 4.75 writes/s, 1.00 fsyncs/s
|
||||||
|
-------------------------------------
|
||||||
|
INSERT BUFFER AND ADAPTIVE HASH INDEX
|
||||||
|
-------------------------------------
|
||||||
|
Ibuf: size 1, free list len 44, seg size 46, 42575 merges
|
||||||
|
merged operations:
|
||||||
|
insert 106806, delete mark 214604, delete 13653
|
||||||
|
discarded operations:
|
||||||
|
insert 0, delete mark 0, delete 0
|
||||||
|
Hash table size 46749559, node heap has 2653 buffer(s)
|
||||||
|
169.46 hash searches/s, 40.24 non-hash searches/s
|
||||||
|
---
|
||||||
|
LOG
|
||||||
|
---
|
||||||
|
Log sequence number 22611157035212
|
||||||
|
Log flushed up to 22611157035212
|
||||||
|
Last checkpoint at 22611148135290
|
||||||
|
Max checkpoint age 434154333
|
||||||
|
Checkpoint age target 420587011
|
||||||
|
Modified age 8899922
|
||||||
|
Checkpoint age 8899922
|
||||||
|
0 pending log writes, 0 pending chkp writes
|
||||||
|
465303593 log i/o's done, 4.75 log i/o's/second
|
||||||
|
----------------------
|
||||||
|
BUFFER POOL AND MEMORY
|
||||||
|
----------------------
|
||||||
|
Total memory allocated 24245174272; in additional pool allocated 0
|
||||||
|
Total memory allocated by read views 2560
|
||||||
|
Internal hash tables (constant factor + variable factor)
|
||||||
|
Adaptive hash index 417467440 (373996472 + 43470968)
|
||||||
|
Page hash 23375608 (buffer pool 0 only)
|
||||||
|
Dictionary cache 141153090 (93500944 + 47652146)
|
||||||
|
File system 1821472 (82672 + 1738800)
|
||||||
|
Lock system 58442168 (58437656 + 4512)
|
||||||
|
Recovery system 0 (0 + 0)
|
||||||
|
Dictionary memory allocated 47652146
|
||||||
|
Buffer pool size 1441791
|
||||||
|
Buffer pool size, bytes 23622303744
|
||||||
|
Free buffers 0
|
||||||
|
Database pages 1439138
|
||||||
|
Old database pages 531224
|
||||||
|
Modified db pages 578
|
||||||
|
Pending reads 0
|
||||||
|
Pending writes: LRU 0, flush list 0, single page 0
|
||||||
|
Pages made young 70145670, not young 0
|
||||||
|
0.00 youngs/s, 0.00 non-youngs/s
|
||||||
|
Pages read 69635295, created 4832854, written 178730753
|
||||||
|
0.00 reads/s, 0.25 creates/s, 0.00 writes/s
|
||||||
|
Buffer pool hit rate 1000 / 1000, young-making rate 0 / 1000 not 0 / 1000
|
||||||
|
Pages read ahead 0.00/s, evicted without access 0.00/s, Random read ahead 0.00/s
|
||||||
|
LRU len: 1439138, unzip_LRU len: 0
|
||||||
|
I/O sum[1103]:cur[0], unzip sum[0]:cur[0]
|
||||||
|
--------------
|
||||||
|
ROW OPERATIONS
|
||||||
|
--------------
|
||||||
|
0 queries inside InnoDB, 0 queries in queue
|
||||||
|
1 read views open inside InnoDB
|
||||||
|
0 transactions active inside InnoDB
|
||||||
|
0 out of 1000 descriptors used
|
||||||
|
---OLDEST VIEW---
|
||||||
|
Normal read view
|
||||||
|
Read view low limit trx n:o 55D01568D
|
||||||
|
Read view up limit trx id 55D01568D
|
||||||
|
Read view low limit trx id 55D01568D
|
||||||
|
Read view individually stored trx ids:
|
||||||
|
-----------------
|
||||||
|
Main thread process no. 1648, id 140570308982528, state: sleeping
|
||||||
|
Number of rows inserted 296824040, updated 207729129, deleted 40014837, read 2455335965266
|
||||||
|
1.75 inserts/s, 1.50 updates/s, 0.50 deletes/s, 177.46 reads/s
|
||||||
|
------------------------
|
||||||
|
LATEST DETECTED DEADLOCK
|
||||||
|
------------------------
|
||||||
|
180227 14:45:46
|
||||||
|
*** (1) TRANSACTION:
|
||||||
|
TRANSACTION 55984EF58, ACTIVE 0 sec inserting
|
||||||
|
mysql tables in use 1, locked 1
|
||||||
|
LOCK WAIT 3 lock struct(s), heap size 376, 2 row lock(s)
|
||||||
|
MySQL thread id 29247292, OS thread handle 0x7fd903731700, query id 2598871462 bm-dell05-publicwww01.bm.int.percona.com 10.10.9.210 root update
|
||||||
|
INSERT INTO semaphore (name, value, expire) VALUES ('variable_init', '557200465a956f1a8822d0.18512229', '1519742747.5575')
|
||||||
|
*** (1) WAITING FOR THIS LOCK TO BE GRANTED:
|
||||||
|
RECORD LOCKS space id 636643 page no 3 n bits 72 index "PRIMARY" of table "percona_com_redesign2015"."semaphore" trx id 55984EF58 lock_mode X locks rec but not gap waiting
|
||||||
|
*** (2) TRANSACTION:
|
||||||
|
TRANSACTION 55984EF59, ACTIVE 0 sec inserting
|
||||||
|
mysql tables in use 1, locked 1
|
||||||
|
3 lock struct(s), heap size 376, 2 row lock(s)
|
||||||
|
MySQL thread id 29247291, OS thread handle 0x7fd9033bf700, query id 2598871463 bm-dell05-publicwww01.bm.int.percona.com 10.10.9.210 root update
|
||||||
|
INSERT INTO semaphore (name, value, expire) VALUES ('variable_init', '1871808655a956f1a882120.81843263', '1519742747.5575')
|
||||||
|
*** (2) HOLDS THE LOCK(S):
|
||||||
|
RECORD LOCKS space id 636643 page no 3 n bits 72 index "PRIMARY" of table "percona_com_redesign2015"."semaphore" trx id 55984EF59 lock mode S locks rec but not gap
|
||||||
|
*** (2) WAITING FOR THIS LOCK TO BE GRANTED:
|
||||||
|
RECORD LOCKS space id 636643 page no 3 n bits 72 index "PRIMARY" of table "percona_com_redesign2015"."semaphore" trx id 55984EF59 lock_mode X locks rec but not gap waiting
|
||||||
|
*** WE ROLL BACK TRANSACTION (2)
|
||||||
|
------------
|
||||||
|
TRANSACTIONS
|
||||||
|
------------
|
||||||
|
Trx id counter 55D015695
|
||||||
|
Purge done for trx's n:o < 55D01567D undo n:o < 0
|
||||||
|
History list length 5198
|
||||||
|
LIST OF TRANSACTIONS FOR EACH SESSION:
|
||||||
|
---TRANSACTION 0, not started
|
||||||
|
MySQL thread id 30893689, OS thread handle 0x7fd902e32700, query id 2738997209 localhost rdba
|
||||||
|
SHOW /*!40100 ENGINE*/ INNODB STATUS
|
||||||
|
---TRANSACTION 0, not started
|
||||||
|
MySQL thread id 30741296, OS thread handle 0x7fd902907700, query id 2738995017 10.10.9.10 rdba
|
||||||
|
---TRANSACTION 55D01565F, not started
|
||||||
|
MySQL thread id 5923113, OS thread handle 0x7fd903b98700, query id 2738997084 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D015660, not started
|
||||||
|
MySQL thread id 5923112, OS thread handle 0x7fd903d20700, query id 2738997087 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D015663, not started
|
||||||
|
MySQL thread id 5922912, OS thread handle 0x7fd903d51700, query id 2738997092 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D01565D, not started
|
||||||
|
MySQL thread id 5922013, OS thread handle 0x7fd9035da700, query id 2738997081 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D015662, not started
|
||||||
|
MySQL thread id 5922000, OS thread handle 0x7fd90366d700, query id 2738997090 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D015590, not started
|
||||||
|
MySQL thread id 689010, OS thread handle 0x7fd9100f5700, query id 2738996241 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D015591, not started
|
||||||
|
MySQL thread id 689009, OS thread handle 0x7fd910157700, query id 2738996244 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D01558E, not started
|
||||||
|
MySQL thread id 689008, OS thread handle 0x7fd91021b700, query id 2738996239 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D01558B, not started
|
||||||
|
MySQL thread id 689006, OS thread handle 0x7fd9104eb700, query id 2738996232 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D01558D, not started
|
||||||
|
MySQL thread id 689004, OS thread handle 0x7fd9101ea700, query id 2738996235 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
----------------------------
|
||||||
|
END OF INNODB MONITOR OUTPUT
|
||||||
|
============================
|
||||||
|
|
||||||
166
src/go/pt-secure-collect/testdata/2018_03_05_13_24_55-innodbstatus2
vendored
Normal file
166
src/go/pt-secure-collect/testdata/2018_03_05_13_24_55-innodbstatus2
vendored
Normal file
@@ -0,0 +1,166 @@
|
|||||||
|
*************************** 1. row ***************************
|
||||||
|
Type: InnoDB
|
||||||
|
Name:
|
||||||
|
Status:
|
||||||
|
=====================================
|
||||||
|
180305 13:25:26 INNODB MONITOR OUTPUT
|
||||||
|
=====================================
|
||||||
|
Per second averages calculated from the last 4 seconds
|
||||||
|
-----------------
|
||||||
|
BACKGROUND THREAD
|
||||||
|
-----------------
|
||||||
|
srv_master_thread loops: 10426647 1_second, 10426519 sleeps, 1039484 10_second, 114409 background, 114408 flush
|
||||||
|
srv_master_thread log flush and writes: 10573097
|
||||||
|
----------
|
||||||
|
SEMAPHORES
|
||||||
|
----------
|
||||||
|
OS WAIT ARRAY INFO: reservation count 36440353, signal count 977828106
|
||||||
|
Mutex spin waits 12033971041, rounds 5762850330, OS waits 2335699
|
||||||
|
RW-shared spins 263735506, rounds 1073399060, OS waits 17684029
|
||||||
|
RW-excl spins 31045742, rounds 1579081354, OS waits 15143737
|
||||||
|
Spin rounds per wait: 0.48 mutex, 4.07 RW-shared, 50.86 RW-excl
|
||||||
|
--------
|
||||||
|
FILE I/O
|
||||||
|
--------
|
||||||
|
I/O thread 0 state: waiting for completed aio requests (insert buffer thread)
|
||||||
|
I/O thread 1 state: waiting for completed aio requests (log thread)
|
||||||
|
I/O thread 2 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 3 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 4 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 5 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 6 state: waiting for completed aio requests (write thread)
|
||||||
|
I/O thread 7 state: waiting for completed aio requests (write thread)
|
||||||
|
I/O thread 8 state: waiting for completed aio requests (write thread)
|
||||||
|
I/O thread 9 state: waiting for completed aio requests (write thread)
|
||||||
|
Pending normal aio reads: 0 [0, 0, 0, 0] , aio writes: 0 [0, 0, 0, 0] ,
|
||||||
|
ibuf aio reads: 0, log i/o's: 0, sync i/o's: 0
|
||||||
|
Pending flushes (fsync) log: 0; buffer pool: 0
|
||||||
|
69640821 OS file reads, 648912256 OS file writes, 26513623 OS fsyncs
|
||||||
|
0.00 reads/s, 0 avg bytes/read, 7.50 writes/s, 0.75 fsyncs/s
|
||||||
|
-------------------------------------
|
||||||
|
INSERT BUFFER AND ADAPTIVE HASH INDEX
|
||||||
|
-------------------------------------
|
||||||
|
Ibuf: size 1, free list len 44, seg size 46, 42575 merges
|
||||||
|
merged operations:
|
||||||
|
insert 106806, delete mark 214604, delete 13653
|
||||||
|
discarded operations:
|
||||||
|
insert 0, delete mark 0, delete 0
|
||||||
|
Hash table size 46749559, node heap has 2653 buffer(s)
|
||||||
|
2154.21 hash searches/s, 777.06 non-hash searches/s
|
||||||
|
---
|
||||||
|
LOG
|
||||||
|
---
|
||||||
|
Log sequence number 22611168022484
|
||||||
|
Log flushed up to 22611167816409
|
||||||
|
Last checkpoint at 22611157596570
|
||||||
|
Max checkpoint age 434154333
|
||||||
|
Checkpoint age target 420587011
|
||||||
|
Modified age 10425914
|
||||||
|
Checkpoint age 10425914
|
||||||
|
0 pending log writes, 0 pending chkp writes
|
||||||
|
465304564 log i/o's done, 7.50 log i/o's/second
|
||||||
|
----------------------
|
||||||
|
BUFFER POOL AND MEMORY
|
||||||
|
----------------------
|
||||||
|
Total memory allocated 24245174272; in additional pool allocated 0
|
||||||
|
Total memory allocated by read views 2560
|
||||||
|
Internal hash tables (constant factor + variable factor)
|
||||||
|
Adaptive hash index 417467440 (373996472 + 43470968)
|
||||||
|
Page hash 23375608 (buffer pool 0 only)
|
||||||
|
Dictionary cache 141153090 (93500944 + 47652146)
|
||||||
|
File system 1821472 (82672 + 1738800)
|
||||||
|
Lock system 58442168 (58437656 + 4512)
|
||||||
|
Recovery system 0 (0 + 0)
|
||||||
|
Dictionary memory allocated 47652146
|
||||||
|
Buffer pool size 1441791
|
||||||
|
Buffer pool size, bytes 23622303744
|
||||||
|
Free buffers 1
|
||||||
|
Database pages 1439137
|
||||||
|
Old database pages 531223
|
||||||
|
Modified db pages 576
|
||||||
|
Pending reads 0
|
||||||
|
Pending writes: LRU 0, flush list 0, single page 0
|
||||||
|
Pages made young 70145670, not young 0
|
||||||
|
0.00 youngs/s, 0.00 non-youngs/s
|
||||||
|
Pages read 69635295, created 4832857, written 178731580
|
||||||
|
0.00 reads/s, 0.00 creates/s, 0.00 writes/s
|
||||||
|
Buffer pool hit rate 1000 / 1000, young-making rate 0 / 1000 not 0 / 1000
|
||||||
|
Pages read ahead 0.00/s, evicted without access 0.00/s, Random read ahead 0.00/s
|
||||||
|
LRU len: 1439137, unzip_LRU len: 0
|
||||||
|
I/O sum[1268]:cur[0], unzip sum[0]:cur[0]
|
||||||
|
--------------
|
||||||
|
ROW OPERATIONS
|
||||||
|
--------------
|
||||||
|
0 queries inside InnoDB, 0 queries in queue
|
||||||
|
1 read views open inside InnoDB
|
||||||
|
0 transactions active inside InnoDB
|
||||||
|
0 out of 1000 descriptors used
|
||||||
|
---OLDEST VIEW---
|
||||||
|
Normal read view
|
||||||
|
Read view low limit trx n:o 55D016078
|
||||||
|
Read view up limit trx id 55D016078
|
||||||
|
Read view low limit trx id 55D016078
|
||||||
|
Read view individually stored trx ids:
|
||||||
|
-----------------
|
||||||
|
Main thread process no. 1648, id 140570308982528, state: sleeping
|
||||||
|
Number of rows inserted 296824578, updated 207729525, deleted 40014897, read 2455341879877
|
||||||
|
3.50 inserts/s, 2.25 updates/s, 1.00 deletes/s, 73224.19 reads/s
|
||||||
|
------------------------
|
||||||
|
LATEST DETECTED DEADLOCK
|
||||||
|
------------------------
|
||||||
|
180227 14:45:46
|
||||||
|
*** (1) TRANSACTION:
|
||||||
|
TRANSACTION 55984EF58, ACTIVE 0 sec inserting
|
||||||
|
mysql tables in use 1, locked 1
|
||||||
|
LOCK WAIT 3 lock struct(s), heap size 376, 2 row lock(s)
|
||||||
|
MySQL thread id 29247292, OS thread handle 0x7fd903731700, query id 2598871462 bm-dell05-publicwww01.bm.int.percona.com 10.10.9.210 root update
|
||||||
|
INSERT INTO semaphore (name, value, expire) VALUES ('variable_init', '557200465a956f1a8822d0.18512229', '1519742747.5575')
|
||||||
|
*** (1) WAITING FOR THIS LOCK TO BE GRANTED:
|
||||||
|
RECORD LOCKS space id 636643 page no 3 n bits 72 index "PRIMARY" of table "percona_com_redesign2015"."semaphore" trx id 55984EF58 lock_mode X locks rec but not gap waiting
|
||||||
|
*** (2) TRANSACTION:
|
||||||
|
TRANSACTION 55984EF59, ACTIVE 0 sec inserting
|
||||||
|
mysql tables in use 1, locked 1
|
||||||
|
3 lock struct(s), heap size 376, 2 row lock(s)
|
||||||
|
MySQL thread id 29247291, OS thread handle 0x7fd9033bf700, query id 2598871463 bm-dell05-publicwww01.bm.int.percona.com 10.10.9.210 root update
|
||||||
|
INSERT INTO semaphore (name, value, expire) VALUES ('variable_init', '1871808655a956f1a882120.81843263', '1519742747.5575')
|
||||||
|
*** (2) HOLDS THE LOCK(S):
|
||||||
|
RECORD LOCKS space id 636643 page no 3 n bits 72 index "PRIMARY" of table "percona_com_redesign2015"."semaphore" trx id 55984EF59 lock mode S locks rec but not gap
|
||||||
|
*** (2) WAITING FOR THIS LOCK TO BE GRANTED:
|
||||||
|
RECORD LOCKS space id 636643 page no 3 n bits 72 index "PRIMARY" of table "percona_com_redesign2015"."semaphore" trx id 55984EF59 lock_mode X locks rec but not gap waiting
|
||||||
|
*** WE ROLL BACK TRANSACTION (2)
|
||||||
|
------------
|
||||||
|
TRANSACTIONS
|
||||||
|
------------
|
||||||
|
Trx id counter 55D016114
|
||||||
|
Purge done for trx's n:o < 55D016070 undo n:o < 0
|
||||||
|
History list length 5648
|
||||||
|
LIST OF TRANSACTIONS FOR EACH SESSION:
|
||||||
|
---TRANSACTION 0, not started
|
||||||
|
MySQL thread id 30893991, OS thread handle 0x7fd9037c4700, query id 2739004581 localhost rdba
|
||||||
|
SHOW /*!40100 ENGINE*/ INNODB STATUS
|
||||||
|
---TRANSACTION 0, not started
|
||||||
|
MySQL thread id 30741296, OS thread handle 0x7fd902907700, query id 2739001334 10.10.9.10 rdba
|
||||||
|
---TRANSACTION 55D015EDC, not started
|
||||||
|
MySQL thread id 5923113, OS thread handle 0x7fd903b98700, query id 2739002770 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D015EDF, not started
|
||||||
|
MySQL thread id 5923112, OS thread handle 0x7fd903d20700, query id 2739002778 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D015EE1, not started
|
||||||
|
MySQL thread id 5922912, OS thread handle 0x7fd903d51700, query id 2739002782 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D015EDE, not started
|
||||||
|
MySQL thread id 5922013, OS thread handle 0x7fd9035da700, query id 2739002773 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D015EE2, not started
|
||||||
|
MySQL thread id 5922000, OS thread handle 0x7fd90366d700, query id 2739002785 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D016056, not started
|
||||||
|
MySQL thread id 689010, OS thread handle 0x7fd9100f5700, query id 2739004138 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D016053, not started
|
||||||
|
MySQL thread id 689009, OS thread handle 0x7fd910157700, query id 2739004129 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D016059, not started
|
||||||
|
MySQL thread id 689008, OS thread handle 0x7fd91021b700, query id 2739004145 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D016055, not started
|
||||||
|
MySQL thread id 689006, OS thread handle 0x7fd9104eb700, query id 2739004133 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D016058, not started
|
||||||
|
MySQL thread id 689004, OS thread handle 0x7fd9101ea700, query id 2739004143 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
----------------------------
|
||||||
|
END OF INNODB MONITOR OUTPUT
|
||||||
|
============================
|
||||||
|
|
||||||
6570
src/go/pt-secure-collect/testdata/2018_03_05_13_24_55-processlist
vendored
Normal file
6570
src/go/pt-secure-collect/testdata/2018_03_05_13_24_55-processlist
vendored
Normal file
File diff suppressed because it is too large
Load Diff
6570
src/go/pt-secure-collect/testdata/2018_03_05_13_24_55-processlist.hostnames.sample
vendored
Normal file
6570
src/go/pt-secure-collect/testdata/2018_03_05_13_24_55-processlist.hostnames.sample
vendored
Normal file
File diff suppressed because it is too large
Load Diff
1
src/go/pt-secure-collect/testdata/2018_03_05_13_24_55-processlist.sample
vendored
Normal file
1
src/go/pt-secure-collect/testdata/2018_03_05_13_24_55-processlist.sample
vendored
Normal file
File diff suppressed because one or more lines are too long
166
src/go/pt-secure-collect/testdata/2018_03_05_13_29_55-innodbstatus1
vendored
Normal file
166
src/go/pt-secure-collect/testdata/2018_03_05_13_29_55-innodbstatus1
vendored
Normal file
@@ -0,0 +1,166 @@
|
|||||||
|
*************************** 1. row ***************************
|
||||||
|
Type: InnoDB
|
||||||
|
Name:
|
||||||
|
Status:
|
||||||
|
=====================================
|
||||||
|
180305 13:29:56 INNODB MONITOR OUTPUT
|
||||||
|
=====================================
|
||||||
|
Per second averages calculated from the last 4 seconds
|
||||||
|
-----------------
|
||||||
|
BACKGROUND THREAD
|
||||||
|
-----------------
|
||||||
|
srv_master_thread loops: 10426915 1_second, 10426787 sleeps, 1039511 10_second, 114409 background, 114408 flush
|
||||||
|
srv_master_thread log flush and writes: 10573367
|
||||||
|
----------
|
||||||
|
SEMAPHORES
|
||||||
|
----------
|
||||||
|
OS WAIT ARRAY INFO: reservation count 36440393, signal count 977828160
|
||||||
|
Mutex spin waits 12033977328, rounds 5762851841, OS waits 2335706
|
||||||
|
RW-shared spins 263735539, rounds 1073399886, OS waits 17684056
|
||||||
|
RW-excl spins 31045745, rounds 1579081554, OS waits 15143742
|
||||||
|
Spin rounds per wait: 0.48 mutex, 4.07 RW-shared, 50.86 RW-excl
|
||||||
|
--------
|
||||||
|
FILE I/O
|
||||||
|
--------
|
||||||
|
I/O thread 0 state: waiting for completed aio requests (insert buffer thread)
|
||||||
|
I/O thread 1 state: waiting for completed aio requests (log thread)
|
||||||
|
I/O thread 2 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 3 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 4 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 5 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 6 state: waiting for completed aio requests (write thread)
|
||||||
|
I/O thread 7 state: waiting for completed aio requests (write thread)
|
||||||
|
I/O thread 8 state: waiting for completed aio requests (write thread)
|
||||||
|
I/O thread 9 state: waiting for completed aio requests (write thread)
|
||||||
|
Pending normal aio reads: 0 [0, 0, 0, 0] , aio writes: 0 [0, 0, 0, 0] ,
|
||||||
|
ibuf aio reads: 0, log i/o's: 0, sync i/o's: 0
|
||||||
|
Pending flushes (fsync) log: 0; buffer pool: 0
|
||||||
|
69640822 OS file reads, 648923500 OS file writes, 26514364 OS fsyncs
|
||||||
|
0.00 reads/s, 0 avg bytes/read, 9.75 writes/s, 1.00 fsyncs/s
|
||||||
|
-------------------------------------
|
||||||
|
INSERT BUFFER AND ADAPTIVE HASH INDEX
|
||||||
|
-------------------------------------
|
||||||
|
Ibuf: size 1, free list len 44, seg size 46, 42575 merges
|
||||||
|
merged operations:
|
||||||
|
insert 106806, delete mark 214604, delete 13653
|
||||||
|
discarded operations:
|
||||||
|
insert 0, delete mark 0, delete 0
|
||||||
|
Hash table size 46749559, node heap has 2653 buffer(s)
|
||||||
|
46.24 hash searches/s, 40.74 non-hash searches/s
|
||||||
|
---
|
||||||
|
LOG
|
||||||
|
---
|
||||||
|
Log sequence number 22611258576197
|
||||||
|
Log flushed up to 22611258571295
|
||||||
|
Last checkpoint at 22611248332855
|
||||||
|
Max checkpoint age 434154333
|
||||||
|
Checkpoint age target 420587011
|
||||||
|
Modified age 10243342
|
||||||
|
Checkpoint age 10243342
|
||||||
|
0 pending log writes, 0 pending chkp writes
|
||||||
|
465310123 log i/o's done, 9.75 log i/o's/second
|
||||||
|
----------------------
|
||||||
|
BUFFER POOL AND MEMORY
|
||||||
|
----------------------
|
||||||
|
Total memory allocated 24245174272; in additional pool allocated 0
|
||||||
|
Total memory allocated by read views 2560
|
||||||
|
Internal hash tables (constant factor + variable factor)
|
||||||
|
Adaptive hash index 417467440 (373996472 + 43470968)
|
||||||
|
Page hash 23375608 (buffer pool 0 only)
|
||||||
|
Dictionary cache 141153090 (93500944 + 47652146)
|
||||||
|
File system 1821472 (82672 + 1738800)
|
||||||
|
Lock system 58442168 (58437656 + 4512)
|
||||||
|
Recovery system 0 (0 + 0)
|
||||||
|
Dictionary memory allocated 47652146
|
||||||
|
Buffer pool size 1441791
|
||||||
|
Buffer pool size, bytes 23622303744
|
||||||
|
Free buffers 1
|
||||||
|
Database pages 1439137
|
||||||
|
Old database pages 531223
|
||||||
|
Modified db pages 594
|
||||||
|
Pending reads 0
|
||||||
|
Pending writes: LRU 0, flush list 0, single page 0
|
||||||
|
Pages made young 70145671, not young 0
|
||||||
|
0.00 youngs/s, 0.00 non-youngs/s
|
||||||
|
Pages read 69635296, created 4832900, written 178737138
|
||||||
|
0.00 reads/s, 0.00 creates/s, 0.00 writes/s
|
||||||
|
Buffer pool hit rate 1000 / 1000, young-making rate 0 / 1000 not 0 / 1000
|
||||||
|
Pages read ahead 0.00/s, evicted without access 0.00/s, Random read ahead 0.00/s
|
||||||
|
LRU len: 1439137, unzip_LRU len: 0
|
||||||
|
I/O sum[1102]:cur[0], unzip sum[0]:cur[0]
|
||||||
|
--------------
|
||||||
|
ROW OPERATIONS
|
||||||
|
--------------
|
||||||
|
0 queries inside InnoDB, 0 queries in queue
|
||||||
|
1 read views open inside InnoDB
|
||||||
|
0 transactions active inside InnoDB
|
||||||
|
0 out of 1000 descriptors used
|
||||||
|
---OLDEST VIEW---
|
||||||
|
Normal read view
|
||||||
|
Read view low limit trx n:o 55D01B7E3
|
||||||
|
Read view up limit trx id 55D01B7E3
|
||||||
|
Read view low limit trx id 55D01B7E3
|
||||||
|
Read view individually stored trx ids:
|
||||||
|
-----------------
|
||||||
|
Main thread process no. 1648, id 140570308982528, state: sleeping
|
||||||
|
Number of rows inserted 296828096, updated 207731732, deleted 40015418, read 2455377947207
|
||||||
|
5.00 inserts/s, 3.50 updates/s, 0.25 deletes/s, 211871.28 reads/s
|
||||||
|
------------------------
|
||||||
|
LATEST DETECTED DEADLOCK
|
||||||
|
------------------------
|
||||||
|
180227 14:45:46
|
||||||
|
*** (1) TRANSACTION:
|
||||||
|
TRANSACTION 55984EF58, ACTIVE 0 sec inserting
|
||||||
|
mysql tables in use 1, locked 1
|
||||||
|
LOCK WAIT 3 lock struct(s), heap size 376, 2 row lock(s)
|
||||||
|
MySQL thread id 29247292, OS thread handle 0x7fd903731700, query id 2598871462 bm-dell05-publicwww01.bm.int.percona.com 10.10.9.210 root update
|
||||||
|
INSERT INTO semaphore (name, value, expire) VALUES ('variable_init', '557200465a956f1a8822d0.18512229', '1519742747.5575')
|
||||||
|
*** (1) WAITING FOR THIS LOCK TO BE GRANTED:
|
||||||
|
RECORD LOCKS space id 636643 page no 3 n bits 72 index "PRIMARY" of table "percona_com_redesign2015"."semaphore" trx id 55984EF58 lock_mode X locks rec but not gap waiting
|
||||||
|
*** (2) TRANSACTION:
|
||||||
|
TRANSACTION 55984EF59, ACTIVE 0 sec inserting
|
||||||
|
mysql tables in use 1, locked 1
|
||||||
|
3 lock struct(s), heap size 376, 2 row lock(s)
|
||||||
|
MySQL thread id 29247291, OS thread handle 0x7fd9033bf700, query id 2598871463 bm-dell05-publicwww01.bm.int.percona.com 10.10.9.210 root update
|
||||||
|
INSERT INTO semaphore (name, value, expire) VALUES ('variable_init', '1871808655a956f1a882120.81843263', '1519742747.5575')
|
||||||
|
*** (2) HOLDS THE LOCK(S):
|
||||||
|
RECORD LOCKS space id 636643 page no 3 n bits 72 index "PRIMARY" of table "percona_com_redesign2015"."semaphore" trx id 55984EF59 lock mode S locks rec but not gap
|
||||||
|
*** (2) WAITING FOR THIS LOCK TO BE GRANTED:
|
||||||
|
RECORD LOCKS space id 636643 page no 3 n bits 72 index "PRIMARY" of table "percona_com_redesign2015"."semaphore" trx id 55984EF59 lock_mode X locks rec but not gap waiting
|
||||||
|
*** WE ROLL BACK TRANSACTION (2)
|
||||||
|
------------
|
||||||
|
TRANSACTIONS
|
||||||
|
------------
|
||||||
|
Trx id counter 55D01B7FF
|
||||||
|
Purge done for trx's n:o < 55D01B7D4 undo n:o < 0
|
||||||
|
History list length 4728
|
||||||
|
LIST OF TRANSACTIONS FOR EACH SESSION:
|
||||||
|
---TRANSACTION 0, not started
|
||||||
|
MySQL thread id 30894959, OS thread handle 0x7fd9037c4700, query id 2739060099 localhost rdba
|
||||||
|
SHOW /*!40100 ENGINE*/ INNODB STATUS
|
||||||
|
---TRANSACTION 0, not started
|
||||||
|
MySQL thread id 30741296, OS thread handle 0x7fd902907700, query id 2739057614 10.10.9.10 rdba
|
||||||
|
---TRANSACTION 55D01B7EF, not started
|
||||||
|
MySQL thread id 5923113, OS thread handle 0x7fd903b98700, query id 2739060064 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D01B7F0, not started
|
||||||
|
MySQL thread id 5923112, OS thread handle 0x7fd903d20700, query id 2739060067 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D01B7EC, not started
|
||||||
|
MySQL thread id 5922912, OS thread handle 0x7fd903d51700, query id 2739060058 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D01B7ED, not started
|
||||||
|
MySQL thread id 5922013, OS thread handle 0x7fd9035da700, query id 2739060061 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D01B7EA, not started
|
||||||
|
MySQL thread id 5922000, OS thread handle 0x7fd90366d700, query id 2739060056 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D01B7FA, not started
|
||||||
|
MySQL thread id 689010, OS thread handle 0x7fd9100f5700, query id 2739060089 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D01B7FC, not started
|
||||||
|
MySQL thread id 689009, OS thread handle 0x7fd910157700, query id 2739060092 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D01B7FD, not started
|
||||||
|
MySQL thread id 689008, OS thread handle 0x7fd91021b700, query id 2739060095 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D01B7F9, not started
|
||||||
|
MySQL thread id 689006, OS thread handle 0x7fd9104eb700, query id 2739060087 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D01B7F7, not started
|
||||||
|
MySQL thread id 689004, OS thread handle 0x7fd9101ea700, query id 2739060084 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
----------------------------
|
||||||
|
END OF INNODB MONITOR OUTPUT
|
||||||
|
============================
|
||||||
|
|
||||||
166
src/go/pt-secure-collect/testdata/2018_03_05_13_29_55-innodbstatus2
vendored
Normal file
166
src/go/pt-secure-collect/testdata/2018_03_05_13_29_55-innodbstatus2
vendored
Normal file
@@ -0,0 +1,166 @@
|
|||||||
|
*************************** 1. row ***************************
|
||||||
|
Type: InnoDB
|
||||||
|
Name:
|
||||||
|
Status:
|
||||||
|
=====================================
|
||||||
|
180305 13:30:26 INNODB MONITOR OUTPUT
|
||||||
|
=====================================
|
||||||
|
Per second averages calculated from the last 4 seconds
|
||||||
|
-----------------
|
||||||
|
BACKGROUND THREAD
|
||||||
|
-----------------
|
||||||
|
srv_master_thread loops: 10426945 1_second, 10426817 sleeps, 1039514 10_second, 114409 background, 114408 flush
|
||||||
|
srv_master_thread log flush and writes: 10573397
|
||||||
|
----------
|
||||||
|
SEMAPHORES
|
||||||
|
----------
|
||||||
|
OS WAIT ARRAY INFO: reservation count 36440428, signal count 977828335
|
||||||
|
Mutex spin waits 12033983140, rounds 5762854799, OS waits 2335733
|
||||||
|
RW-shared spins 263735563, rounds 1073400093, OS waits 17684061
|
||||||
|
RW-excl spins 31045750, rounds 1579081835, OS waits 15143745
|
||||||
|
Spin rounds per wait: 0.48 mutex, 4.07 RW-shared, 50.86 RW-excl
|
||||||
|
--------
|
||||||
|
FILE I/O
|
||||||
|
--------
|
||||||
|
I/O thread 0 state: waiting for completed aio requests (insert buffer thread)
|
||||||
|
I/O thread 1 state: waiting for completed aio requests (log thread)
|
||||||
|
I/O thread 2 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 3 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 4 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 5 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 6 state: waiting for completed aio requests (write thread)
|
||||||
|
I/O thread 7 state: waiting for completed aio requests (write thread)
|
||||||
|
I/O thread 8 state: waiting for completed aio requests (write thread)
|
||||||
|
I/O thread 9 state: waiting for completed aio requests (write thread)
|
||||||
|
Pending normal aio reads: 0 [0, 0, 0, 0] , aio writes: 0 [0, 0, 0, 0] ,
|
||||||
|
ibuf aio reads: 0, log i/o's: 0, sync i/o's: 0
|
||||||
|
Pending flushes (fsync) log: 0; buffer pool: 0
|
||||||
|
69640822 OS file reads, 648932213 OS file writes, 26514419 OS fsyncs
|
||||||
|
0.00 reads/s, 0 avg bytes/read, 49.49 writes/s, 1.00 fsyncs/s
|
||||||
|
-------------------------------------
|
||||||
|
INSERT BUFFER AND ADAPTIVE HASH INDEX
|
||||||
|
-------------------------------------
|
||||||
|
Ibuf: size 1, free list len 44, seg size 46, 42575 merges
|
||||||
|
merged operations:
|
||||||
|
insert 106806, delete mark 214604, delete 13653
|
||||||
|
discarded operations:
|
||||||
|
insert 0, delete mark 0, delete 0
|
||||||
|
Hash table size 46749559, node heap has 2653 buffer(s)
|
||||||
|
11618.10 hash searches/s, 1378.66 non-hash searches/s
|
||||||
|
---
|
||||||
|
LOG
|
||||||
|
---
|
||||||
|
Log sequence number 22611271140958
|
||||||
|
Log flushed up to 22611270996231
|
||||||
|
Last checkpoint at 22611252175617
|
||||||
|
Max checkpoint age 434154333
|
||||||
|
Checkpoint age target 420587011
|
||||||
|
Modified age 18965341
|
||||||
|
Checkpoint age 18965341
|
||||||
|
0 pending log writes, 0 pending chkp writes
|
||||||
|
465318567 log i/o's done, 49.49 log i/o's/second
|
||||||
|
----------------------
|
||||||
|
BUFFER POOL AND MEMORY
|
||||||
|
----------------------
|
||||||
|
Total memory allocated 24245174272; in additional pool allocated 0
|
||||||
|
Total memory allocated by read views 2560
|
||||||
|
Internal hash tables (constant factor + variable factor)
|
||||||
|
Adaptive hash index 417467440 (373996472 + 43470968)
|
||||||
|
Page hash 23375608 (buffer pool 0 only)
|
||||||
|
Dictionary cache 141153090 (93500944 + 47652146)
|
||||||
|
File system 1821472 (82672 + 1738800)
|
||||||
|
Lock system 58442168 (58437656 + 4512)
|
||||||
|
Recovery system 0 (0 + 0)
|
||||||
|
Dictionary memory allocated 47652146
|
||||||
|
Buffer pool size 1441791
|
||||||
|
Buffer pool size, bytes 23622303744
|
||||||
|
Free buffers 1
|
||||||
|
Database pages 1439137
|
||||||
|
Old database pages 531223
|
||||||
|
Modified db pages 1085
|
||||||
|
Pending reads 0
|
||||||
|
Pending writes: LRU 0, flush list 0, single page 0
|
||||||
|
Pages made young 70145671, not young 0
|
||||||
|
0.00 youngs/s, 0.00 non-youngs/s
|
||||||
|
Pages read 69635296, created 4832914, written 178737400
|
||||||
|
0.00 reads/s, 0.25 creates/s, 0.00 writes/s
|
||||||
|
Buffer pool hit rate 1000 / 1000, young-making rate 0 / 1000 not 0 / 1000
|
||||||
|
Pages read ahead 0.00/s, evicted without access 0.00/s, Random read ahead 0.00/s
|
||||||
|
LRU len: 1439137, unzip_LRU len: 0
|
||||||
|
I/O sum[702]:cur[0], unzip sum[0]:cur[0]
|
||||||
|
--------------
|
||||||
|
ROW OPERATIONS
|
||||||
|
--------------
|
||||||
|
0 queries inside InnoDB, 0 queries in queue
|
||||||
|
1 read views open inside InnoDB
|
||||||
|
0 transactions active inside InnoDB
|
||||||
|
0 out of 1000 descriptors used
|
||||||
|
---OLDEST VIEW---
|
||||||
|
Normal read view
|
||||||
|
Read view low limit trx n:o 55D02049F
|
||||||
|
Read view up limit trx id 55D02049F
|
||||||
|
Read view low limit trx id 55D02049F
|
||||||
|
Read view individually stored trx ids:
|
||||||
|
-----------------
|
||||||
|
Main thread process no. 1648, id 140570308982528, state: sleeping
|
||||||
|
Number of rows inserted 296829308, updated 207739319, deleted 40015463, read 2455401692192
|
||||||
|
28.24 inserts/s, 21.74 updates/s, 0.75 deletes/s, 164406.15 reads/s
|
||||||
|
------------------------
|
||||||
|
LATEST DETECTED DEADLOCK
|
||||||
|
------------------------
|
||||||
|
180227 14:45:46
|
||||||
|
*** (1) TRANSACTION:
|
||||||
|
TRANSACTION 55984EF58, ACTIVE 0 sec inserting
|
||||||
|
mysql tables in use 1, locked 1
|
||||||
|
LOCK WAIT 3 lock struct(s), heap size 376, 2 row lock(s)
|
||||||
|
MySQL thread id 29247292, OS thread handle 0x7fd903731700, query id 2598871462 bm-dell05-publicwww01.bm.int.percona.com 10.10.9.210 root update
|
||||||
|
INSERT INTO semaphore (name, value, expire) VALUES ('variable_init', '557200465a956f1a8822d0.18512229', '1519742747.5575')
|
||||||
|
*** (1) WAITING FOR THIS LOCK TO BE GRANTED:
|
||||||
|
RECORD LOCKS space id 636643 page no 3 n bits 72 index "PRIMARY" of table "percona_com_redesign2015"."semaphore" trx id 55984EF58 lock_mode X locks rec but not gap waiting
|
||||||
|
*** (2) TRANSACTION:
|
||||||
|
TRANSACTION 55984EF59, ACTIVE 0 sec inserting
|
||||||
|
mysql tables in use 1, locked 1
|
||||||
|
3 lock struct(s), heap size 376, 2 row lock(s)
|
||||||
|
MySQL thread id 29247291, OS thread handle 0x7fd9033bf700, query id 2598871463 bm-dell05-publicwww01.bm.int.percona.com 10.10.9.210 root update
|
||||||
|
INSERT INTO semaphore (name, value, expire) VALUES ('variable_init', '1871808655a956f1a882120.81843263', '1519742747.5575')
|
||||||
|
*** (2) HOLDS THE LOCK(S):
|
||||||
|
RECORD LOCKS space id 636643 page no 3 n bits 72 index "PRIMARY" of table "percona_com_redesign2015"."semaphore" trx id 55984EF59 lock mode S locks rec but not gap
|
||||||
|
*** (2) WAITING FOR THIS LOCK TO BE GRANTED:
|
||||||
|
RECORD LOCKS space id 636643 page no 3 n bits 72 index "PRIMARY" of table "percona_com_redesign2015"."semaphore" trx id 55984EF59 lock_mode X locks rec but not gap waiting
|
||||||
|
*** WE ROLL BACK TRANSACTION (2)
|
||||||
|
------------
|
||||||
|
TRANSACTIONS
|
||||||
|
------------
|
||||||
|
Trx id counter 55D02054F
|
||||||
|
Purge done for trx's n:o < 55D02049F undo n:o < 0
|
||||||
|
History list length 2173
|
||||||
|
LIST OF TRANSACTIONS FOR EACH SESSION:
|
||||||
|
---TRANSACTION 0, not started
|
||||||
|
MySQL thread id 30895279, OS thread handle 0x7fd903857700, query id 2739081619 localhost rdba
|
||||||
|
SHOW /*!40100 ENGINE*/ INNODB STATUS
|
||||||
|
---TRANSACTION 0, not started
|
||||||
|
MySQL thread id 30741296, OS thread handle 0x7fd902907700, query id 2739075683 10.10.9.10 rdba
|
||||||
|
---TRANSACTION 55D020530, not started
|
||||||
|
MySQL thread id 5923113, OS thread handle 0x7fd903b98700, query id 2739081571 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D02052A, not started
|
||||||
|
MySQL thread id 5923112, OS thread handle 0x7fd903d20700, query id 2739081559 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D02052B, not started
|
||||||
|
MySQL thread id 5922912, OS thread handle 0x7fd903d51700, query id 2739081564 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D02052E, not started
|
||||||
|
MySQL thread id 5922013, OS thread handle 0x7fd9035da700, query id 2739081567 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D020528, not started
|
||||||
|
MySQL thread id 5922000, OS thread handle 0x7fd90366d700, query id 2739081557 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D0204BB, not started
|
||||||
|
MySQL thread id 689010, OS thread handle 0x7fd9100f5700, query id 2739081344 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D0204BD, not started
|
||||||
|
MySQL thread id 689009, OS thread handle 0x7fd910157700, query id 2739081347 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D0204B3, not started
|
||||||
|
MySQL thread id 689008, OS thread handle 0x7fd91021b700, query id 2739081333 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D0204B8, not started
|
||||||
|
MySQL thread id 689006, OS thread handle 0x7fd9104eb700, query id 2739081341 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D0204B6, not started
|
||||||
|
MySQL thread id 689004, OS thread handle 0x7fd9101ea700, query id 2739081337 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
----------------------------
|
||||||
|
END OF INNODB MONITOR OUTPUT
|
||||||
|
============================
|
||||||
|
|
||||||
166
src/go/pt-secure-collect/testdata/2018_03_05_13_34_55-innodbstatus1
vendored
Normal file
166
src/go/pt-secure-collect/testdata/2018_03_05_13_34_55-innodbstatus1
vendored
Normal file
@@ -0,0 +1,166 @@
|
|||||||
|
*************************** 1. row ***************************
|
||||||
|
Type: InnoDB
|
||||||
|
Name:
|
||||||
|
Status:
|
||||||
|
=====================================
|
||||||
|
180305 13:34:56 INNODB MONITOR OUTPUT
|
||||||
|
=====================================
|
||||||
|
Per second averages calculated from the last 4 seconds
|
||||||
|
-----------------
|
||||||
|
BACKGROUND THREAD
|
||||||
|
-----------------
|
||||||
|
srv_master_thread loops: 10427213 1_second, 10427085 sleeps, 1039541 10_second, 114412 background, 114411 flush
|
||||||
|
srv_master_thread log flush and writes: 10573668
|
||||||
|
----------
|
||||||
|
SEMAPHORES
|
||||||
|
----------
|
||||||
|
OS WAIT ARRAY INFO: reservation count 36440480, signal count 977828396
|
||||||
|
Mutex spin waits 12033991389, rounds 5762858481, OS waits 2335752
|
||||||
|
RW-shared spins 263735596, rounds 1073400964, OS waits 17684090
|
||||||
|
RW-excl spins 31045751, rounds 1579081972, OS waits 15143749
|
||||||
|
Spin rounds per wait: 0.48 mutex, 4.07 RW-shared, 50.86 RW-excl
|
||||||
|
--------
|
||||||
|
FILE I/O
|
||||||
|
--------
|
||||||
|
I/O thread 0 state: waiting for completed aio requests (insert buffer thread)
|
||||||
|
I/O thread 1 state: waiting for completed aio requests (log thread)
|
||||||
|
I/O thread 2 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 3 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 4 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 5 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 6 state: waiting for completed aio requests (write thread)
|
||||||
|
I/O thread 7 state: waiting for completed aio requests (write thread)
|
||||||
|
I/O thread 8 state: waiting for completed aio requests (write thread)
|
||||||
|
I/O thread 9 state: waiting for completed aio requests (write thread)
|
||||||
|
Pending normal aio reads: 0 [0, 0, 0, 0] , aio writes: 0 [0, 0, 0, 0] ,
|
||||||
|
ibuf aio reads: 0, log i/o's: 0, sync i/o's: 0
|
||||||
|
Pending flushes (fsync) log: 0; buffer pool: 0
|
||||||
|
69640822 OS file reads, 648944242 OS file writes, 26515183 OS fsyncs
|
||||||
|
0.00 reads/s, 0 avg bytes/read, 66.48 writes/s, 5.50 fsyncs/s
|
||||||
|
-------------------------------------
|
||||||
|
INSERT BUFFER AND ADAPTIVE HASH INDEX
|
||||||
|
-------------------------------------
|
||||||
|
Ibuf: size 1, free list len 44, seg size 46, 42575 merges
|
||||||
|
merged operations:
|
||||||
|
insert 106806, delete mark 214604, delete 13653
|
||||||
|
discarded operations:
|
||||||
|
insert 0, delete mark 0, delete 0
|
||||||
|
Hash table size 46749559, node heap has 2654 buffer(s)
|
||||||
|
114.72 hash searches/s, 118.72 non-hash searches/s
|
||||||
|
---
|
||||||
|
LOG
|
||||||
|
---
|
||||||
|
Log sequence number 22611346881333
|
||||||
|
Log flushed up to 22611346881333
|
||||||
|
Last checkpoint at 22611346009655
|
||||||
|
Max checkpoint age 434154333
|
||||||
|
Checkpoint age target 420587011
|
||||||
|
Modified age 871678
|
||||||
|
Checkpoint age 871678
|
||||||
|
0 pending log writes, 0 pending chkp writes
|
||||||
|
465324837 log i/o's done, 10.25 log i/o's/second
|
||||||
|
----------------------
|
||||||
|
BUFFER POOL AND MEMORY
|
||||||
|
----------------------
|
||||||
|
Total memory allocated 24245174272; in additional pool allocated 0
|
||||||
|
Total memory allocated by read views 2560
|
||||||
|
Internal hash tables (constant factor + variable factor)
|
||||||
|
Adaptive hash index 417483824 (373996472 + 43487352)
|
||||||
|
Page hash 23375608 (buffer pool 0 only)
|
||||||
|
Dictionary cache 141153090 (93500944 + 47652146)
|
||||||
|
File system 1821472 (82672 + 1738800)
|
||||||
|
Lock system 58442168 (58437656 + 4512)
|
||||||
|
Recovery system 0 (0 + 0)
|
||||||
|
Dictionary memory allocated 47652146
|
||||||
|
Buffer pool size 1441791
|
||||||
|
Buffer pool size, bytes 23622303744
|
||||||
|
Free buffers 1
|
||||||
|
Database pages 1439136
|
||||||
|
Old database pages 531223
|
||||||
|
Modified db pages 257
|
||||||
|
Pending reads 0
|
||||||
|
Pending writes: LRU 0, flush list 0, single page 0
|
||||||
|
Pages made young 70145671, not young 0
|
||||||
|
0.00 youngs/s, 0.00 non-youngs/s
|
||||||
|
Pages read 69635296, created 4832946, written 178743030
|
||||||
|
0.00 reads/s, 0.00 creates/s, 54.99 writes/s
|
||||||
|
Buffer pool hit rate 1000 / 1000, young-making rate 0 / 1000 not 0 / 1000
|
||||||
|
Pages read ahead 0.00/s, evicted without access 0.00/s, Random read ahead 0.00/s
|
||||||
|
LRU len: 1439136, unzip_LRU len: 0
|
||||||
|
I/O sum[1376]:cur[0], unzip sum[0]:cur[0]
|
||||||
|
--------------
|
||||||
|
ROW OPERATIONS
|
||||||
|
--------------
|
||||||
|
0 queries inside InnoDB, 0 queries in queue
|
||||||
|
1 read views open inside InnoDB
|
||||||
|
0 transactions active inside InnoDB
|
||||||
|
0 out of 1000 descriptors used
|
||||||
|
---OLDEST VIEW---
|
||||||
|
Normal read view
|
||||||
|
Read view low limit trx n:o 55D025F9F
|
||||||
|
Read view up limit trx id 55D025F9F
|
||||||
|
Read view low limit trx id 55D025F9F
|
||||||
|
Read view individually stored trx ids:
|
||||||
|
-----------------
|
||||||
|
Main thread process no. 1648, id 140570308982528, state: sleeping
|
||||||
|
Number of rows inserted 296832979, updated 207741812, deleted 40015812, read 2455438612265
|
||||||
|
5.25 inserts/s, 4.00 updates/s, 0.00 deletes/s, 141586.85 reads/s
|
||||||
|
------------------------
|
||||||
|
LATEST DETECTED DEADLOCK
|
||||||
|
------------------------
|
||||||
|
180227 14:45:46
|
||||||
|
*** (1) TRANSACTION:
|
||||||
|
TRANSACTION 55984EF58, ACTIVE 0 sec inserting
|
||||||
|
mysql tables in use 1, locked 1
|
||||||
|
LOCK WAIT 3 lock struct(s), heap size 376, 2 row lock(s)
|
||||||
|
MySQL thread id 29247292, OS thread handle 0x7fd903731700, query id 2598871462 bm-dell05-publicwww01.bm.int.percona.com 10.10.9.210 root update
|
||||||
|
INSERT INTO semaphore (name, value, expire) VALUES ('variable_init', '557200465a956f1a8822d0.18512229', '1519742747.5575')
|
||||||
|
*** (1) WAITING FOR THIS LOCK TO BE GRANTED:
|
||||||
|
RECORD LOCKS space id 636643 page no 3 n bits 72 index "PRIMARY" of table "percona_com_redesign2015"."semaphore" trx id 55984EF58 lock_mode X locks rec but not gap waiting
|
||||||
|
*** (2) TRANSACTION:
|
||||||
|
TRANSACTION 55984EF59, ACTIVE 0 sec inserting
|
||||||
|
mysql tables in use 1, locked 1
|
||||||
|
3 lock struct(s), heap size 376, 2 row lock(s)
|
||||||
|
MySQL thread id 29247291, OS thread handle 0x7fd9033bf700, query id 2598871463 bm-dell05-publicwww01.bm.int.percona.com 10.10.9.210 root update
|
||||||
|
INSERT INTO semaphore (name, value, expire) VALUES ('variable_init', '1871808655a956f1a882120.81843263', '1519742747.5575')
|
||||||
|
*** (2) HOLDS THE LOCK(S):
|
||||||
|
RECORD LOCKS space id 636643 page no 3 n bits 72 index "PRIMARY" of table "percona_com_redesign2015"."semaphore" trx id 55984EF59 lock mode S locks rec but not gap
|
||||||
|
*** (2) WAITING FOR THIS LOCK TO BE GRANTED:
|
||||||
|
RECORD LOCKS space id 636643 page no 3 n bits 72 index "PRIMARY" of table "percona_com_redesign2015"."semaphore" trx id 55984EF59 lock_mode X locks rec but not gap waiting
|
||||||
|
*** WE ROLL BACK TRANSACTION (2)
|
||||||
|
------------
|
||||||
|
TRANSACTIONS
|
||||||
|
------------
|
||||||
|
Trx id counter 55D025F9F
|
||||||
|
Purge done for trx's n:o < 55D025F9F undo n:o < 0
|
||||||
|
History list length 3865
|
||||||
|
LIST OF TRANSACTIONS FOR EACH SESSION:
|
||||||
|
---TRANSACTION 0, not started
|
||||||
|
MySQL thread id 30896336, OS thread handle 0x7fd902e32700, query id 2739141923 localhost rdba
|
||||||
|
SHOW /*!40100 ENGINE*/ INNODB STATUS
|
||||||
|
---TRANSACTION 0, not started
|
||||||
|
MySQL thread id 30741296, OS thread handle 0x7fd902907700, query id 2739140489 10.10.9.10 rdba
|
||||||
|
---TRANSACTION 55D025F95, not started
|
||||||
|
MySQL thread id 5923113, OS thread handle 0x7fd903b98700, query id 2739141887 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D025F90, not started
|
||||||
|
MySQL thread id 5923112, OS thread handle 0x7fd903d20700, query id 2739141878 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D025F96, not started
|
||||||
|
MySQL thread id 5922912, OS thread handle 0x7fd903d51700, query id 2739141889 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D025F92, not started
|
||||||
|
MySQL thread id 5922013, OS thread handle 0x7fd9035da700, query id 2739141880 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D025F93, not started
|
||||||
|
MySQL thread id 5922000, OS thread handle 0x7fd90366d700, query id 2739141883 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D025F9C, not started
|
||||||
|
MySQL thread id 689010, OS thread handle 0x7fd9100f5700, query id 2739141914 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D025F99, not started
|
||||||
|
MySQL thread id 689009, OS thread handle 0x7fd910157700, query id 2739141908 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D025F98, not started
|
||||||
|
MySQL thread id 689008, OS thread handle 0x7fd91021b700, query id 2739141905 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D025F9A, not started
|
||||||
|
MySQL thread id 689006, OS thread handle 0x7fd9104eb700, query id 2739141911 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D025F9D, not started
|
||||||
|
MySQL thread id 689004, OS thread handle 0x7fd9101ea700, query id 2739141916 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
----------------------------
|
||||||
|
END OF INNODB MONITOR OUTPUT
|
||||||
|
============================
|
||||||
|
|
||||||
168
src/go/pt-secure-collect/testdata/2018_03_05_13_34_55-innodbstatus2
vendored
Normal file
168
src/go/pt-secure-collect/testdata/2018_03_05_13_34_55-innodbstatus2
vendored
Normal file
@@ -0,0 +1,168 @@
|
|||||||
|
*************************** 1. row ***************************
|
||||||
|
Type: InnoDB
|
||||||
|
Name:
|
||||||
|
Status:
|
||||||
|
=====================================
|
||||||
|
180305 13:35:26 INNODB MONITOR OUTPUT
|
||||||
|
=====================================
|
||||||
|
Per second averages calculated from the last 4 seconds
|
||||||
|
-----------------
|
||||||
|
BACKGROUND THREAD
|
||||||
|
-----------------
|
||||||
|
srv_master_thread loops: 10427243 1_second, 10427115 sleeps, 1039544 10_second, 114412 background, 114411 flush
|
||||||
|
srv_master_thread log flush and writes: 10573698
|
||||||
|
----------
|
||||||
|
SEMAPHORES
|
||||||
|
----------
|
||||||
|
OS WAIT ARRAY INFO: reservation count 36440485, signal count 977828405
|
||||||
|
Mutex spin waits 12033992104, rounds 5762858828, OS waits 2335754
|
||||||
|
RW-shared spins 263735599, rounds 1073401054, OS waits 17684093
|
||||||
|
RW-excl spins 31045752, rounds 1579081980, OS waits 15143749
|
||||||
|
Spin rounds per wait: 0.48 mutex, 4.07 RW-shared, 50.86 RW-excl
|
||||||
|
--------
|
||||||
|
FILE I/O
|
||||||
|
--------
|
||||||
|
I/O thread 0 state: waiting for completed aio requests (insert buffer thread)
|
||||||
|
I/O thread 1 state: waiting for completed aio requests (log thread)
|
||||||
|
I/O thread 2 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 3 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 4 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 5 state: waiting for completed aio requests (read thread)
|
||||||
|
I/O thread 6 state: waiting for completed aio requests (write thread)
|
||||||
|
I/O thread 7 state: waiting for completed aio requests (write thread)
|
||||||
|
I/O thread 8 state: waiting for completed aio requests (write thread)
|
||||||
|
I/O thread 9 state: waiting for completed aio requests (write thread)
|
||||||
|
Pending normal aio reads: 0 [0, 0, 0, 0] , aio writes: 0 [0, 0, 0, 0] ,
|
||||||
|
ibuf aio reads: 0, log i/o's: 0, sync i/o's: 0
|
||||||
|
Pending flushes (fsync) log: 0; buffer pool: 0
|
||||||
|
69640822 OS file reads, 648945544 OS file writes, 26515245 OS fsyncs
|
||||||
|
0.00 reads/s, 0 avg bytes/read, 62.23 writes/s, 3.75 fsyncs/s
|
||||||
|
-------------------------------------
|
||||||
|
INSERT BUFFER AND ADAPTIVE HASH INDEX
|
||||||
|
-------------------------------------
|
||||||
|
Ibuf: size 1, free list len 44, seg size 46, 42575 merges
|
||||||
|
merged operations:
|
||||||
|
insert 106806, delete mark 214604, delete 13653
|
||||||
|
discarded operations:
|
||||||
|
insert 0, delete mark 0, delete 0
|
||||||
|
Hash table size 46749559, node heap has 2654 buffer(s)
|
||||||
|
378.41 hash searches/s, 191.70 non-hash searches/s
|
||||||
|
---
|
||||||
|
LOG
|
||||||
|
---
|
||||||
|
Log sequence number 22611355523170
|
||||||
|
Log flushed up to 22611355523170
|
||||||
|
Last checkpoint at 22611346878065
|
||||||
|
Max checkpoint age 434154333
|
||||||
|
Checkpoint age target 420587011
|
||||||
|
Modified age 8645105
|
||||||
|
Checkpoint age 8645105
|
||||||
|
0 pending log writes, 0 pending chkp writes
|
||||||
|
465325667 log i/o's done, 6.00 log i/o's/second
|
||||||
|
----------------------
|
||||||
|
BUFFER POOL AND MEMORY
|
||||||
|
----------------------
|
||||||
|
Total memory allocated 24245174272; in additional pool allocated 0
|
||||||
|
Total memory allocated by read views 2560
|
||||||
|
Internal hash tables (constant factor + variable factor)
|
||||||
|
Adaptive hash index 417483824 (373996472 + 43487352)
|
||||||
|
Page hash 23375608 (buffer pool 0 only)
|
||||||
|
Dictionary cache 141153090 (93500944 + 47652146)
|
||||||
|
File system 1821472 (82672 + 1738800)
|
||||||
|
Lock system 58442544 (58437656 + 4888)
|
||||||
|
Recovery system 0 (0 + 0)
|
||||||
|
Dictionary memory allocated 47652146
|
||||||
|
Buffer pool size 1441791
|
||||||
|
Buffer pool size, bytes 23622303744
|
||||||
|
Free buffers 1
|
||||||
|
Database pages 1439136
|
||||||
|
Old database pages 531223
|
||||||
|
Modified db pages 378
|
||||||
|
Pending reads 0
|
||||||
|
Pending writes: LRU 0, flush list 0, single page 0
|
||||||
|
Pages made young 70145671, not young 0
|
||||||
|
0.00 youngs/s, 0.00 non-youngs/s
|
||||||
|
Pages read 69635296, created 4832952, written 178743491
|
||||||
|
0.00 reads/s, 0.00 creates/s, 54.99 writes/s
|
||||||
|
Buffer pool hit rate 1000 / 1000, young-making rate 0 / 1000 not 0 / 1000
|
||||||
|
Pages read ahead 0.00/s, evicted without access 0.00/s, Random read ahead 0.00/s
|
||||||
|
LRU len: 1439136, unzip_LRU len: 0
|
||||||
|
I/O sum[901]:cur[0], unzip sum[0]:cur[0]
|
||||||
|
--------------
|
||||||
|
ROW OPERATIONS
|
||||||
|
--------------
|
||||||
|
0 queries inside InnoDB, 0 queries in queue
|
||||||
|
1 read views open inside InnoDB
|
||||||
|
0 transactions active inside InnoDB
|
||||||
|
0 out of 1000 descriptors used
|
||||||
|
---OLDEST VIEW---
|
||||||
|
Normal read view
|
||||||
|
Read view low limit trx n:o 55D026BA0
|
||||||
|
Read view up limit trx id 55D026BA0
|
||||||
|
Read view low limit trx id 55D026BA0
|
||||||
|
Read view individually stored trx ids:
|
||||||
|
-----------------
|
||||||
|
Main thread process no. 1648, id 140570308982528, state: sleeping
|
||||||
|
Number of rows inserted 296833449, updated 207742148, deleted 40015845, read 2455447075033
|
||||||
|
4.00 inserts/s, 1.25 updates/s, 1.00 deletes/s, 770.56 reads/s
|
||||||
|
------------------------
|
||||||
|
LATEST DETECTED DEADLOCK
|
||||||
|
------------------------
|
||||||
|
180227 14:45:46
|
||||||
|
*** (1) TRANSACTION:
|
||||||
|
TRANSACTION 55984EF58, ACTIVE 0 sec inserting
|
||||||
|
mysql tables in use 1, locked 1
|
||||||
|
LOCK WAIT 3 lock struct(s), heap size 376, 2 row lock(s)
|
||||||
|
MySQL thread id 29247292, OS thread handle 0x7fd903731700, query id 2598871462 bm-dell05-publicwww01.bm.int.percona.com 10.10.9.210 root update
|
||||||
|
INSERT INTO semaphore (name, value, expire) VALUES ('variable_init', '557200465a956f1a8822d0.18512229', '1519742747.5575')
|
||||||
|
*** (1) WAITING FOR THIS LOCK TO BE GRANTED:
|
||||||
|
RECORD LOCKS space id 636643 page no 3 n bits 72 index "PRIMARY" of table "percona_com_redesign2015"."semaphore" trx id 55984EF58 lock_mode X locks rec but not gap waiting
|
||||||
|
*** (2) TRANSACTION:
|
||||||
|
TRANSACTION 55984EF59, ACTIVE 0 sec inserting
|
||||||
|
mysql tables in use 1, locked 1
|
||||||
|
3 lock struct(s), heap size 376, 2 row lock(s)
|
||||||
|
MySQL thread id 29247291, OS thread handle 0x7fd9033bf700, query id 2598871463 bm-dell05-publicwww01.bm.int.percona.com 10.10.9.210 root update
|
||||||
|
INSERT INTO semaphore (name, value, expire) VALUES ('variable_init', '1871808655a956f1a882120.81843263', '1519742747.5575')
|
||||||
|
*** (2) HOLDS THE LOCK(S):
|
||||||
|
RECORD LOCKS space id 636643 page no 3 n bits 72 index "PRIMARY" of table "percona_com_redesign2015"."semaphore" trx id 55984EF59 lock mode S locks rec but not gap
|
||||||
|
*** (2) WAITING FOR THIS LOCK TO BE GRANTED:
|
||||||
|
RECORD LOCKS space id 636643 page no 3 n bits 72 index "PRIMARY" of table "percona_com_redesign2015"."semaphore" trx id 55984EF59 lock_mode X locks rec but not gap waiting
|
||||||
|
*** WE ROLL BACK TRANSACTION (2)
|
||||||
|
------------
|
||||||
|
TRANSACTIONS
|
||||||
|
------------
|
||||||
|
Trx id counter 55D026BA1
|
||||||
|
Purge done for trx's n:o < 55D026B83 undo n:o < 0
|
||||||
|
History list length 3988
|
||||||
|
LIST OF TRANSACTIONS FOR EACH SESSION:
|
||||||
|
---TRANSACTION 0, not started
|
||||||
|
MySQL thread id 30896632, OS thread handle 0x7fd903e15700, query id 2739149745 localhost rdba
|
||||||
|
SHOW /*!40100 ENGINE*/ INNODB STATUS
|
||||||
|
---TRANSACTION 0, not started
|
||||||
|
MySQL thread id 30896630, OS thread handle 0x7fd903857700, query id 2739149740 bm-dell05-publicwww01.bm.int.percona.com 10.10.9.210 root
|
||||||
|
---TRANSACTION 0, not started
|
||||||
|
MySQL thread id 30741296, OS thread handle 0x7fd902907700, query id 2739146469 10.10.9.10 rdba
|
||||||
|
---TRANSACTION 55D026B7E, not started
|
||||||
|
MySQL thread id 5923113, OS thread handle 0x7fd903b98700, query id 2739149495 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D026B80, not started
|
||||||
|
MySQL thread id 5923112, OS thread handle 0x7fd903d20700, query id 2739149497 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D026B81, not started
|
||||||
|
MySQL thread id 5922912, OS thread handle 0x7fd903d51700, query id 2739149500 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D026B7B, not started
|
||||||
|
MySQL thread id 5922013, OS thread handle 0x7fd9035da700, query id 2739149488 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D026B7D, not started
|
||||||
|
MySQL thread id 5922000, OS thread handle 0x7fd90366d700, query id 2739149491 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D026AE8, not started
|
||||||
|
MySQL thread id 689010, OS thread handle 0x7fd9100f5700, query id 2739149086 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D026AE6, not started
|
||||||
|
MySQL thread id 689009, OS thread handle 0x7fd910157700, query id 2739149083 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D026AE9, not started
|
||||||
|
MySQL thread id 689008, OS thread handle 0x7fd91021b700, query id 2739149090 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D026AEC, not started
|
||||||
|
MySQL thread id 689006, OS thread handle 0x7fd9104eb700, query id 2739149096 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
---TRANSACTION 55D026AEB, not started
|
||||||
|
MySQL thread id 689004, OS thread handle 0x7fd9101ea700, query id 2739149093 www-docker01.bm.int.percona.com 10.10.9.22 version_check
|
||||||
|
----------------------------
|
||||||
|
END OF INNODB MONITOR OUTPUT
|
||||||
|
============================
|
||||||
|
|
||||||
2
src/go/pt-secure-collect/testdata/pt-mysql-summary
vendored
Executable file
2
src/go/pt-secure-collect/testdata/pt-mysql-summary
vendored
Executable file
@@ -0,0 +1,2 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
echo "a dummy output for pt-mysql-summary"
|
||||||
9
src/go/pt-secure-collect/testdata/pt-stalk
vendored
Executable file
9
src/go/pt-secure-collect/testdata/pt-stalk
vendored
Executable file
@@ -0,0 +1,9 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# pt-stalk --no-stalk --iterations=2 --sleep=30 --host=127.0.0.1 --dest=/home/karl/data_collection_2018-04-17_15_41_29 --port=5721 --user=msandbox --password=********
|
||||||
|
# arg[0] [1] [2] [3] [4] [5] [6] [7] [8]
|
||||||
|
|
||||||
|
preffix="--dest="
|
||||||
|
dest_dir=${5#$prefix}; #Remove prefix
|
||||||
|
|
||||||
|
echo "a dummy output for pt-stalk" > ${dest_dir}/dummy.log
|
||||||
3
src/go/pt-secure-collect/testdata/pt-summary
vendored
Executable file
3
src/go/pt-secure-collect/testdata/pt-summary
vendored
Executable file
@@ -0,0 +1,3 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
echo "a dummy output for pt-summary"
|
||||||
2029
src/go/pt-secure-collect/testdata/slow_80.log
vendored
Executable file
2029
src/go/pt-secure-collect/testdata/slow_80.log
vendored
Executable file
File diff suppressed because it is too large
Load Diff
BIN
src/go/pt-secure-collect/testdata/slow_80.log.sample
vendored
Normal file
BIN
src/go/pt-secure-collect/testdata/slow_80.log.sample
vendored
Normal file
Binary file not shown.
Reference in New Issue
Block a user