mirror of
https://github.com/percona/percona-toolkit.git
synced 2026-02-28 02:07:54 +08:00
PT-2156 Fix tests for lib, part2 (#650)
* PT-2156 - Fix tests for lib Fixed tests, broken by putting fix for PT-1554 into the library code, mostly by updating checksums. Added AutoCommit option for test lib/QueryReview.t Reset SQL mode to empty to allow TableChunker to work with invalid and zero dates * PT-2156 - Fix tests for lib Adjusted t/lib/SchemaIterator.t to work with version 8.0 * PT-2156 - Fix tests for lib Adjusted b/t/lib/Processlist.t, so it reflects fix for PT-981 * PT-2156 - Fix tests for lib Adjusted t/lib/HTTP/Micro.t, so it works with different order of parameters, returned at v.percona.com/ Adjusted test for empty files in t/lib/bash/collect.t Disabled mysqladmin debug test in t/lib/bash/collect.t, because of PT-2242 * PT-2156 - Fix tests for lib Added LC_NUMERRIC=POSIX into t/lib/bash/report_system_info.sh, so reports in tests are not environment-dependent Updated expected results in t/lib/bash/report_system_info.sh, so they reflect new information, collected by pt-summary * PT-2156 - Fix tests for lib - Improved fix for PT-76, so it handles inline comments - Added test case for PT-76 - Improved fix for PT-1720, so it ignores unrecognizable option only if it comes from the toolkit-wide files and still errors out if wrong option was passed via command line or the tool-specific option file. * PT-2156 - Fix tests for lib - Improved fix for PT-2102, so it finds running instance configuration file using PID and also does not break t/lib/bash/report_mysql_info.t test - Removed unnecessary copy-paste from t/pt-mysql-summary/pt-2102.t test - Adjusted number of collected files in t/lib/bash/collect_mysql_info.sh * PT-2156 - Fix tests for lib - Fix for PT-1543 and MyRocks collection were originally put only into lib/bash/report_mysql_info.shthat broke the logic of collecting data first, then formatting report from this data. This, in its turn, broke test t/lib/bash/report_mysql_info.sh/t/lib/bash/report_mysql_info.t, because CMD_MYSQL is not defined in this library. I rewrote these fixes, so they follow original logic of the tool (pt-mysql-summary) - Added tests for keyring plugin, encrypted tables, and MyRocks for t/lib/bash/report_mysql_info.sh and t/pt-mysql-summary/pt-mysql-summary_encryption.t * PT-2156 - Fix tests for lib - Added FLUSH TABLES to t/lib/bash/collect.sh, so it does not fail opentables tests if run when more than 1000 tables open in the sandbox environment - Changed number of expected sample files to reflect keyring colletion file * PT-2156 - Fix tests for lib - Added skip to some tests in lib that file when run with PXC, because not supported to work with PXC - Adjusted configuration files for PXC, so they allow LOAD DATA/SELECT INTO OUTFILE commands - Adjusted data samples, so they do not depend on auto increment values - Fixed lib tests, failing with PXC * PT-2156 - Fix tests for lib Updated tests for pt-online-schema-change, so they work with PXC and skipped if designed for semi-synchronous replication setup * PT-2156 - Fix tests for lib - Added cluster-specific samples for t/lib/SchemaIterator.t - Removed extra debugging print from t/pt-table-checksum/pt-1728.t * PT-2156 - Fix tests for lib Evgeniy's review on July 20 * Update t/lib/TableSyncer.t Removed comment left after debugging * Update t/pt-mysql-summary/pt-mysql-summary_encryption.t Removed extra debug output * PT-2156 - Fix tests for lib Tabs to spaces
This commit is contained in:
Sveta Smirnova
@@ -57,9 +57,21 @@ collect_mysqld_instances () {
|
||||
find_my_cnf_file() {
|
||||
local file="$1"
|
||||
local port="${2:-""}"
|
||||
local pid_file="${3:-""}"
|
||||
|
||||
local cnf_file=""
|
||||
|
||||
if [ "$pid_file" ]; then
|
||||
local pid=$(cat "$pid_file")
|
||||
cnf_file="$(grep --max-count 1 -E "^\s*$pid\s+" "$file" \
|
||||
| awk 'BEGIN{RS=" "; FS="=";} $1 ~ /--defaults-file/ { print $2; }')"
|
||||
|
||||
if [ -n "$cnf_file" ]; then
|
||||
echo "$cnf_file"
|
||||
return
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$port" ]; then
|
||||
# Find the cnf file for the specific port.
|
||||
cnf_file="$(grep --max-count 1 "/mysqld.*--port=$port" "$file" \
|
||||
@@ -69,12 +81,11 @@ find_my_cnf_file() {
|
||||
echo "$cnf_file"
|
||||
return
|
||||
fi
|
||||
else
|
||||
cnf_file="$(grep --max-count 1 '/mysqld' "$file" \
|
||||
| awk 'BEGIN{RS=" "; FS="=";} $1 ~ /--defaults-file/ { print $2; }')"
|
||||
fi
|
||||
|
||||
cnf_file="$(grep --max-count 1 '/mysqld' "$file" \
|
||||
| awk 'BEGIN{RS=" "; FS="=";} $1 ~ /--defaults-file/ { print $2; }')"
|
||||
|
||||
|
||||
if [ -z "$cnf_file" ]; then
|
||||
# Cannot autodetect config file, try common locations.
|
||||
if [ -e "/etc/my.cnf" ]; then
|
||||
@@ -180,6 +191,46 @@ collect_internal_vars () {
|
||||
i=$(($i + 1))
|
||||
done < "$mysqld_executables"
|
||||
fi
|
||||
|
||||
# jemalloc info
|
||||
local JEMALLOC_STATUS=''
|
||||
local GENERAL_JEMALLOC_STATUS=0
|
||||
local JEMALLOC_LOCATION=''
|
||||
|
||||
for pid in $(pidof mysqld); do
|
||||
grep -qc jemalloc /proc/${pid}/environ || ldd $(which mysqld) 2>/dev/null | grep -qc jemalloc
|
||||
jemalloc_status=$?
|
||||
if [ $jemalloc_status = 1 ]; then
|
||||
echo "pt-summary-internal-jemalloc_enabled_for_pid_${pid} 0"
|
||||
else
|
||||
echo "pt-summary-internal-jemalloc_enabled_for_pid_${pid} 1"
|
||||
GENERAL_JEMALLOC_STATUS=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $GENERAL_JEMALLOC_STATUS -eq 1 ]; then
|
||||
JEMALLOC_LOCATION=$(find /usr/lib64/ /usr/lib/x86_64-linux-gnu /usr/lib -name "libjemalloc.*" 2>/dev/null | head -n 1)
|
||||
echo "pt-summary-internal-jemalloc_location ${JEMALLOC_LOCATION}"
|
||||
fi
|
||||
}
|
||||
|
||||
collect_keyring_plugins() {
|
||||
$CMD_MYSQL $EXT_ARGV --table -ss -e 'SELECT PLUGIN_NAME, PLUGIN_STATUS FROM INFORMATION_SCHEMA.PLUGINS WHERE PLUGIN_NAME LIKE "keyring%";'
|
||||
}
|
||||
|
||||
collect_encrypted_tables() {
|
||||
$CMD_MYSQL $EXT_ARGV --table -ss -e "SELECT TABLE_SCHEMA, TABLE_NAME, CREATE_OPTIONS FROM INFORMATION_SCHEMA.TABLES WHERE CREATE_OPTIONS LIKE '%ENCRYPTION=_Y_%';"
|
||||
}
|
||||
|
||||
collect_encrypted_tablespaces() {
|
||||
local version="$1"
|
||||
# I_S.INNODB_[SYS_]TABLESPACES has a "flag" field. Encrypted tablespace has bit 14 set. You can check it with "flag & 8192".
|
||||
# And seems like MySQL is capable of bitwise operations. https://dev.mysql.com/doc/refman/5.7/en/bit-functions.html
|
||||
if [ "$version" '<' "8.0" ]; then
|
||||
$CMD_MYSQL $EXT_ARGV --table -ss -e "SELECT SPACE, NAME, SPACE_TYPE from INFORMATION_SCHEMA.INNODB_SYS_TABLESPACES where FLAG&8192 = 8192;"
|
||||
else
|
||||
$CMD_MYSQL $EXT_ARGV --table -ss -e "SELECT SPACE, NAME, SPACE_TYPE from INFORMATION_SCHEMA.INNODB_TABLESPACES where FLAG&8192 = 8192;"
|
||||
fi
|
||||
}
|
||||
|
||||
# Uses mysqldump and dumps the results to FILE.
|
||||
@@ -246,6 +297,7 @@ collect_mysql_info () {
|
||||
collect_mysql_processlist > "$dir/mysql-processlist"
|
||||
collect_mysql_users > "$dir/mysql-users"
|
||||
collect_mysql_roles > "$dir/mysql-roles"
|
||||
collect_keyring_plugins > "$dir/keyring-plugins"
|
||||
|
||||
collect_mysqld_instances "$dir/mysql-variables" > "$dir/mysqld-instances"
|
||||
collect_mysqld_executables "$dir/mysqld-instances" > "$dir/mysqld-executables"
|
||||
@@ -261,16 +313,16 @@ collect_mysql_info () {
|
||||
local current_time="$($CMD_MYSQL $EXT_ARGV -ss -e \
|
||||
"SELECT LEFT(NOW() - INTERVAL ${uptime} SECOND, 16)")"
|
||||
|
||||
local port="$(get_var port "$dir/mysql-variables")"
|
||||
local cnf_file="$(find_my_cnf_file "$dir/mysqld-instances" ${port})"
|
||||
|
||||
[ -e "$cnf_file" ] && cat "$cnf_file" > "$dir/mysql-config-file"
|
||||
|
||||
local pid_file="$(get_var "pid_file" "$dir/mysql-variables")"
|
||||
local pid_file_exists=""
|
||||
[ -e "${pid_file}" ] && pid_file_exists=1
|
||||
echo "pt-summary-internal-pid_file_exists $pid_file_exists" >> "$dir/mysql-variables"
|
||||
|
||||
local port="$(get_var port "$dir/mysql-variables")"
|
||||
local cnf_file="$(find_my_cnf_file "$dir/mysqld-instances" ${port} ${pid_file})"
|
||||
|
||||
[ -e "$cnf_file" ] && cat "$cnf_file" > "$dir/mysql-config-file"
|
||||
|
||||
# TODO: Do these require a file of their own?
|
||||
echo "pt-summary-internal-current_time $current_time" >> "$dir/mysql-variables"
|
||||
echo "pt-summary-internal-Config_File_path $cnf_file" >> "$dir/mysql-variables"
|
||||
@@ -283,6 +335,13 @@ collect_mysql_info () {
|
||||
get_mysqldump_for "${trg_arg}" "$dbs" > "$dir/mysqldump"
|
||||
fi
|
||||
|
||||
# encrypted tables and tablespaces
|
||||
if [ "${OPT_LIST_ENCRYPTED_TABLES}" = 'yes' ]; then
|
||||
local mysql_version="$(get_var version "$dir/mysql-variables")"
|
||||
collect_encrypted_tables > "$dir/encrypted-tables"
|
||||
collect_encrypted_tablespaces ${mysql_version} > "$dir/encrypted-tablespaces"
|
||||
fi
|
||||
|
||||
# TODO: gather this data in the same format as normal: TS line, stats
|
||||
(
|
||||
sleep $OPT_SLEEP
|
||||
|
||||
@@ -56,6 +56,8 @@ OPT_VERSION="" # If --version was specified
|
||||
OPT_HELP="" # If --help was specified
|
||||
OPT_ASK_PASS="" # If --ask-pass was specified
|
||||
PO_DIR="" # Directory with program option spec files
|
||||
GLOBAL_CONFIG=0 # We ignore non-recognized options in global configs
|
||||
# and return error for user-defined configs and command line
|
||||
|
||||
# Sub: usage
|
||||
# Print usage (--help) and list the program's options.
|
||||
@@ -215,10 +217,16 @@ parse_options() {
|
||||
_parse_config_files "$user_config_file"
|
||||
done
|
||||
else
|
||||
_parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf"
|
||||
GLOBAL_CONFIG=1
|
||||
_parse_config_files "/etc/percona-toolkit/percona-toolkit.conf"
|
||||
GLOBAL_CONFIG=0
|
||||
_parse_config_files "/etc/percona-toolkit/$TOOL.conf"
|
||||
# conditional in case $HOME isn't set; e.g. tool launched from init
|
||||
if [ "${HOME:-}" ]; then
|
||||
_parse_config_files "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf"
|
||||
GLOBAL_CONFIG=1
|
||||
_parse_config_files "$HOME/.percona-toolkit.conf"
|
||||
GLOBAL_CONFIG=0
|
||||
_parse_config_files "$HOME/.$TOOL.conf"
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -348,7 +356,7 @@ _parse_config_files() {
|
||||
|
||||
# Strip leading and trailing spaces, and spaces around the first =,
|
||||
# and end-of-line # comments.
|
||||
config_opt="$(echo "$config_opt" | sed -e 's/^ *//g' -e 's/ *$//g' -e 's/[ ]*=[ ]*/=/' -e 's/[ ]+#.*$//')"
|
||||
config_opt="$(echo "$config_opt" | sed -e 's/^ *//g' -e 's/ *$//g' -e 's/[ ]*=[ ]*/=/' -e 's/\s[ ]*#.*$//')"
|
||||
|
||||
# Skip blank lines.
|
||||
[ "$config_opt" = "" ] && continue
|
||||
@@ -468,11 +476,14 @@ _parse_command_line() {
|
||||
else
|
||||
spec=$(grep "^short form:-$opt\$" "$PT_TMPDIR"/po/* | cut -d ':' -f 1)
|
||||
if [ -z "$spec" ]; then
|
||||
# Not all programs uses the same options and since these options can be stored
|
||||
# in a common config file, we need to skip general options not used by a particular
|
||||
# program
|
||||
# option_error "Unknown option: $real_opt"
|
||||
continue
|
||||
if [ $GLOBAL_CONFIG -eq 1 ]; then
|
||||
# Not all programs uses the same options and since these options can be stored
|
||||
# in a common config file, we need to skip general options not used by a particular
|
||||
# program
|
||||
continue
|
||||
else
|
||||
option_error "Unknown option: $real_opt"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
@@ -107,26 +107,6 @@ get_plugin_status () {
|
||||
echo ${status:-"Not found"}
|
||||
}
|
||||
|
||||
collect_keyring_plugins() {
|
||||
$CMD_MYSQL $EXT_ARGV --table -ss -e 'SELECT PLUGIN_NAME, PLUGIN_STATUS FROM INFORMATION_SCHEMA.PLUGINS WHERE PLUGIN_NAME LIKE "keyring%";'
|
||||
}
|
||||
|
||||
collect_encrypted_tables() {
|
||||
$CMD_MYSQL $EXT_ARGV --table -ss -e "SELECT TABLE_SCHEMA, TABLE_NAME, CREATE_OPTIONS FROM INFORMATION_SCHEMA.TABLES WHERE CREATE_OPTIONS LIKE '%ENCRYPTION=\"Y\"%';"
|
||||
}
|
||||
|
||||
collect_encrypted_tablespaces() {
|
||||
local version="$1"
|
||||
# I_S.INNODB_[SYS_]TABLESPACES has a "flag" field. Encrypted tablespace has bit 14 set. You can check it with "flag & 8192".
|
||||
# And seems like MySQL is capable of bitwise operations. https://dev.mysql.com/doc/refman/5.7/en/bit-functions.html
|
||||
if [ "$version" '<' "8.0" ]; then
|
||||
$CMD_MYSQL $EXT_ARGV --table -ss -e "SELECT SPACE, NAME, SPACE_TYPE from INFORMATION_SCHEMA.INNODB_SYS_TABLESPACES where FLAG&8192 = 8192;"
|
||||
else
|
||||
$CMD_MYSQL $EXT_ARGV --table -ss -e "SELECT SPACE, NAME, SPACE_TYPE from INFORMATION_SCHEMA.INNODB_TABLESPACES where FLAG&8192 = 8192;"
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
# ##############################################################################
|
||||
# Functions for parsing specific files and getting desired info from them.
|
||||
# These are called from within main() and are separated so they can be tested
|
||||
@@ -1292,30 +1272,31 @@ parse_wsrep_provider_options () {
|
||||
}
|
||||
|
||||
report_jemalloc_enabled() {
|
||||
local JEMALLOC_STATUS=''
|
||||
local GENERAL_JEMALLOC_STATUS=0
|
||||
local JEMALLOC_LOCATION=''
|
||||
local instances_file="$1"
|
||||
local variables_file="$2"
|
||||
local GENERAL_JEMALLOC_STATUS=0
|
||||
|
||||
for pid in $(grep '/mysqld ' "$instances_file" | awk '{print $1;}'); do
|
||||
local jemalloc_status="$(get_var "pt-summary-internal-jemalloc_enabled_for_pid_${pid}" "${variables_file}")"
|
||||
if [ -z $jemalloc_status ]; then
|
||||
continue
|
||||
elif [ $jemalloc_status = 0 ]; then
|
||||
echo "jemalloc is not enabled in mysql config for process with id ${pid}"
|
||||
else
|
||||
echo "jemalloc enabled in mysql config for process with id ${pid}"
|
||||
GENERAL_JEMALLOC_STATUS=1
|
||||
fi
|
||||
done
|
||||
|
||||
for pid in $(pidof mysqld); do
|
||||
grep -qc jemalloc /proc/${pid}/environ || ldd $(which mysqld) 2>/dev/null | grep -qc jemalloc
|
||||
jemalloc_status=$?
|
||||
if [ $jemalloc_status = 1 ]; then
|
||||
echo "jemalloc is not enabled in mysql config for process with id ${pid}"
|
||||
else
|
||||
echo "jemalloc enabled in mysql config for process with id ${pid}"
|
||||
GENERAL_JEMALLOC_STATUS=1
|
||||
fi
|
||||
done
|
||||
if [ $GENERAL_JEMALLOC_STATUS -eq 1 ]; then
|
||||
local jemalloc_location="$(get_var "pt-summary-internal-jemalloc_location" "${variables_file}")"
|
||||
if [ -n "$jemalloc_location" ]; then
|
||||
echo "Using jemalloc from $jemalloc_location"
|
||||
else
|
||||
echo "Jemalloc library not found"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $GENERAL_JEMALLOC_STATUS -eq 1 ]; then
|
||||
JEMALLOC_LOCATION=$(find /usr/lib64/ /usr/lib/x86_64-linux-gnu /usr/lib -name "libjemalloc.*" 2>/dev/null | head -n 1)
|
||||
if [ -z "$JEMALLOC_LOCATION" ]; then
|
||||
echo "Jemalloc library not found"
|
||||
else
|
||||
echo "Using jemalloc from $JEMALLOC_LOCATION"
|
||||
fi
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
report_mysql_summary () {
|
||||
@@ -1566,7 +1547,7 @@ report_mysql_summary () {
|
||||
fi
|
||||
fi
|
||||
|
||||
local has_rocksdb=$($CMD_MYSQL $EXT_ARGV -ss -e 'SHOW ENGINES' 2>/dev/null | grep -i 'rocksdb')
|
||||
local has_rocksdb=$(cat $dir/mysql-plugins | grep -i 'rocksdb.*active.*storage engine')
|
||||
if [ ! -z "$has_rocksdb" ]; then
|
||||
section "RocksDB"
|
||||
section_rocksdb "$dir/mysql-variables" "$dir/mysql-status"
|
||||
@@ -1600,13 +1581,17 @@ report_mysql_summary () {
|
||||
fi
|
||||
|
||||
section "Encryption"
|
||||
local keyring_plugins="$(collect_keyring_plugins)"
|
||||
local keyring_plugins=""
|
||||
local encrypted_tables=""
|
||||
local encrypted_tablespaces=""
|
||||
if [ "${OPT_LIST_ENCRYPTED_TABLES}" = 'yes' ]; then
|
||||
local mysql_version="$(get_var version "$dir/mysql-variables")"
|
||||
encrypted_tables="$(collect_encrypted_tables)"
|
||||
encrypted_tablespaces="$(collect_encrypted_tablespaces ${mysql_version})"
|
||||
if [ -s "$dir/keyring-plugins" ]; then
|
||||
keyring_plugins="$(cat $dir/keyring-plugins)"
|
||||
fi
|
||||
if [ -s "$dir/encrypted-tables" ]; then
|
||||
encrypted_tables="$(cat $dir/encrypted-tables)"
|
||||
fi
|
||||
if [ -s "$dir/encrypted-tablespaces" ]; then
|
||||
encrypted_tablespaces="$(cat $dir/encrypted-tablespaces)"
|
||||
fi
|
||||
|
||||
format_keyring_plugins "$keyring_plugins" "$encrypted_tables"
|
||||
@@ -1652,7 +1637,7 @@ report_mysql_summary () {
|
||||
fi
|
||||
|
||||
section "Memory management library"
|
||||
report_jemalloc_enabled
|
||||
report_jemalloc_enabled "$dir/mysqld-instances" "$dir/mysql-variables"
|
||||
|
||||
# Make sure that we signal the end of the tool's output.
|
||||
section "The End"
|
||||
|
||||
Reference in New Issue
Block a user