percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-13 14:39:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed sharded/unsharded coll stats

Browse Source
This commit is contained in:
Carlos Salguero
2017-01-05 23:52:03 -03:00
parent 56b8587715
commit 3fd93b4f32
5 changed files with 152 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/go/Makefile
View File

@@ -1,13 +1,18 @@
GO := go GO := go
pkgs = $(shell $(GO) list ./... | grep -v /vendor/) pkgs = $(shell $(GO) list ./... | grep -v /vendor/)
VERSION=$(git describe --tags) VERSION=$(shell git describe --tags)
BUILD=$(date +%FT%T%z) BUILD=$(shell date +%FT%T%z)
PREFIX=$(shell pwd) PREFIX=$(shell pwd)
BIN_DIR=$(shell git rev-parse --show-toplevel)/bin BIN_DIR=$(shell git rev-parse --show-toplevel)/bin
LDFLAGS="-w -s -X main.Version=${VERSION} -X main.Build=${BUILD}" LDFLAGS="-X main.Version=${VERSION} -X main.Build=${BUILD}"
build-all:
@echo ">> building all binaries in ${BIN_DIR}. OS & platform will be appended to the file name"
@echo "LDFLAGS: ${LDFLAGS}"
@gox -ldflags=${LDFLAGS} -osarch="linux/amd64 linux/386 darwin/amd64" -output=${BIN_DIR}"/{{.Dir}}_{{.OS}}_{{.Arch}}" $(pkgs)
all: format gox test all: format gox test
style: style:
@@ -26,10 +31,6 @@ vet:
@echo ">> vetting code" @echo ">> vetting code"
@$(GO) vet $(pkgs) @$(GO) vet $(pkgs)
build-all:
@echo ">> building all binaries in $(BIN_DIR). OS & platform will be appended to the file name"
@gox -ldflags ${LDFLAGS} -osarch="linux/amd64 linux/386 darwin/amd64" -output=${BIN_DIR}"/{{.Dir}}_{{.OS}}_{{.Arch}}" $(pkgs)
linux-amd64: linux-amd64:
@echo ">> building linux/amd64 binaries in $(BIN_DIR)" @echo ">> building linux/amd64 binaries in $(BIN_DIR)"
gox -ldflags ${LDFLAGS} -osarch="linux/amd64" -output=${BIN_DIR}"/{{.Dir}}" $(pkgs) gox -ldflags ${LDFLAGS} -osarch="linux/amd64" -output=${BIN_DIR}"/{{.Dir}}" $(pkgs)

46
src/go/mongolib/proto/collstats.go Normal file
View File

@@ -0,0 +1,46 @@
package proto
type ShardStas struct {
Ns string `json:"ns"`
Count int64 `json:"count"`
Size int64 `json:"size"`
AvgObjSize int64 `json:"avgObjSize"`
NumExtents int64 `json:"numExtents"`
StorageSize int64 `json:"storageSize"`
LastExtentSize int64 `json:"lastExtentSize"`
PaddingFactor int64 `json:"paddingFactor"`
PaddingFactorNote string `json:"paddingFactorNote"`
UserFlags int64 `json:"userFlags"`
Capped bool `json:"capped"`
Nindexes int64 `json:"nindexes"`
IndexDetails struct {
} `json:"indexDetails"`
TotalIndexSize int64 `json:"totalIndexSize"`
IndexSizes struct {
ID int64 `json:"_id_"`
IDHashed int64 `json:"_id_hashed"`
} `json:"indexSizes"`
Ok int `json:"ok"`
}
type CollStats struct {
Sharded bool `json:"sharded"`
PaddingFactorNote string `json:"paddingFactorNote"`
UserFlags int64 `json:"userFlags"`
Capped bool `json:"capped"`
Ns string `json:"ns"`
Count int64 `json:"count"`
NumExtents int64 `json:"numExtents"`
Size int64 `json:"size"`
StorageSize int64 `json:"storageSize"`
TotalIndexSize int64 `json:"totalIndexSize"`
IndexSizes struct {
ID int `json:"_id_"`
IDHashed int `json:"_id_hashed"`
} `json:"indexSizes"`
AvgObjSize int64 `json:"avgObjSize"`
Nindexes int64 `json:"nindexes"`
Nchunks int64 `json:"nchunks"`
Shards map[string]ShardStas `json:"shards"`
Ok int64 `json:"ok"`
}

180
src/go/pt-mongodb-summary/main.go
View File

@@ -184,7 +184,7 @@ func main() {
} }
var sampleCount int64 = 5 var sampleCount int64 = 5
var sampleRate time.Duration = 1 // in seconds var sampleRate time.Duration = 1 * time.Second // in seconds
if rops, err := GetOpCountersStats(session, sampleCount, sampleRate); err != nil { if rops, err := GetOpCountersStats(session, sampleCount, sampleRate); err != nil {
log.Printf("[Error] cannot get Opcounters stats: %v\n", err) log.Printf("[Error] cannot get Opcounters stats: %v\n", err)
} else { } else {
@@ -364,21 +364,29 @@ func GetClusterwideInfo(session pmgo.SessionManager) (*clusterwideInfo, error) {
TotalDBsCount: len(databases.Databases), TotalDBsCount: len(databases.Databases),
} }
configDB := session.DB("config")
for _, db := range databases.Databases { for _, db := range databases.Databases {
collections, err := session.DB(db.Name).CollectionNames() collections, err := session.DB(db.Name).CollectionNames()
if err != nil { if err != nil {
continue continue
} }
cwi.TotalCollectionsCount += len(collections) cwi.TotalCollectionsCount += len(collections)
if len(db.Shards) == 1 { for _, collName := range collections {
cwi.UnshardedDataSize += db.SizeOnDisk var collStats proto.CollStats
continue err := session.DB(db.Name).Run(bson.M{"collStats": collName}, &collStats)
if err != nil {
continue
}
if collStats.Sharded {
cwi.ShardedDataSize += collStats.Size
cwi.ShardedColsCount++
continue
}
cwi.UnshardedDataSize += collStats.Size
cwi.UnshardedColsCount++
} }
cwi.ShardedDataSize += db.SizeOnDisk
colsCount, _ := configDB.C("collections").Find(bson.M{"_id": bson.RegEx{"^" + db.Name, ""}}).Count()
cwi.ShardedColsCount += colsCount
} }
cwi.UnshardedColsCount = cwi.TotalCollectionsCount - cwi.ShardedColsCount cwi.UnshardedColsCount = cwi.TotalCollectionsCount - cwi.ShardedColsCount
@@ -389,14 +397,14 @@ func GetClusterwideInfo(session pmgo.SessionManager) (*clusterwideInfo, error) {
} }
func sizeAndUnit(size int64) (float64, string) { func sizeAndUnit(size int64) (float64, string) {
unit := []string{"KB", "MB", "GB", "TB"} unit := []string{"bytes", "KB", "MB", "GB", "TB"}
idx := 0 idx := 0
newSize := float64(size) newSize := float64(size)
for newSize > 1024 { for newSize > 1024 {
newSize /= 1024 newSize /= 1024
idx++ idx++
} }
newSize = float64(int64(newSize*100) / 1000) newSize = float64(int64(newSize*100)) / 100
return newSize, unit[idx] return newSize, unit[idx]
} }
@@ -476,11 +484,14 @@ func getNodeType(session pmgo.SessionManager) (string, error) {
func GetOpCountersStats(session pmgo.SessionManager, count int64, sleep time.Duration) (*opCounters, error) { func GetOpCountersStats(session pmgo.SessionManager, count int64, sleep time.Duration) (*opCounters, error) {
oc := &opCounters{} oc := &opCounters{}
previousoc := &opCounters{} prevOpCount := &opCounters{}
ss := proto.ServerStatus{} ss := proto.ServerStatus{}
delta := proto.ServerStatus{
Opcounters: &proto.OpcountStats{},
}
ticker := time.NewTicker(sleep) ticker := time.NewTicker(sleep)
for i := int64(0); i < count; i++ { for i := int64(0); i < count+1; i++ {
<-ticker.C <-ticker.C
err := session.DB("admin").Run(bson.D{{"serverStatus", 1}, {"recordStats", 1}}, &ss) err := session.DB("admin").Run(bson.D{{"serverStatus", 1}, {"recordStats", 1}}, &ss)
if err != nil { if err != nil {
@@ -488,111 +499,116 @@ func GetOpCountersStats(session pmgo.SessionManager, count int64, sleep time.Dur
} }
if i == 0 { if i == 0 {
previousoc.Command.Total = ss.Opcounters.Command prevOpCount.Command.Total = ss.Opcounters.Command
previousoc.Delete.Total = ss.Opcounters.Delete prevOpCount.Delete.Total = ss.Opcounters.Delete
previousoc.GetMore.Total = ss.Opcounters.GetMore prevOpCount.GetMore.Total = ss.Opcounters.GetMore
previousoc.Insert.Total = ss.Opcounters.Insert prevOpCount.Insert.Total = ss.Opcounters.Insert
previousoc.Query.Total = ss.Opcounters.Query prevOpCount.Query.Total = ss.Opcounters.Query
previousoc.Update.Total = ss.Opcounters.Update prevOpCount.Update.Total = ss.Opcounters.Update
continue continue
} }
ss.Opcounters.Command -= previousoc.Command.Total delta.Opcounters.Command = ss.Opcounters.Command - prevOpCount.Command.Total
ss.Opcounters.Delete -= previousoc.Delete.Total delta.Opcounters.Delete = ss.Opcounters.Delete - prevOpCount.Delete.Total
ss.Opcounters.GetMore -= previousoc.GetMore.Total delta.Opcounters.GetMore = ss.Opcounters.GetMore - prevOpCount.GetMore.Total
ss.Opcounters.Insert -= previousoc.Insert.Total delta.Opcounters.Insert = ss.Opcounters.Insert - prevOpCount.Insert.Total
ss.Opcounters.Query -= previousoc.Query.Total delta.Opcounters.Query = ss.Opcounters.Query - prevOpCount.Query.Total
ss.Opcounters.Update -= previousoc.Update.Total delta.Opcounters.Update = ss.Opcounters.Update - prevOpCount.Update.Total
// Be careful. This cannot be item[0] because we need value - prev_value // Be careful. This cannot be item[0] because we need: value - prev_value
// and at pos 0 there is no prev value // and at pos 0 there is no prev value
if i == 1 { if i == 1 {
oc.Command.Max = ss.Opcounters.Command oc.Command.Max = delta.Opcounters.Command
oc.Command.Min = ss.Opcounters.Command oc.Command.Min = delta.Opcounters.Command
oc.Delete.Max = ss.Opcounters.Delete
oc.Delete.Min = ss.Opcounters.Delete oc.Delete.Max = delta.Opcounters.Delete
oc.GetMore.Max = ss.Opcounters.GetMore oc.Delete.Min = delta.Opcounters.Delete
oc.GetMore.Min = ss.Opcounters.GetMore
oc.Insert.Max = ss.Opcounters.Insert oc.GetMore.Max = delta.Opcounters.GetMore
oc.Insert.Min = ss.Opcounters.Insert oc.GetMore.Min = delta.Opcounters.GetMore
oc.Query.Max = ss.Opcounters.Query
oc.Query.Min = ss.Opcounters.Query oc.Insert.Max = delta.Opcounters.Insert
oc.Update.Max = ss.Opcounters.Update oc.Insert.Min = delta.Opcounters.Insert
oc.Update.Min = ss.Opcounters.Update
oc.Query.Max = delta.Opcounters.Query
oc.Query.Min = delta.Opcounters.Query
oc.Update.Max = delta.Opcounters.Update
oc.Update.Min = delta.Opcounters.Update
} }
// Insert // Insert --------------------------------------
if ss.Opcounters.Insert > oc.Insert.Max { if delta.Opcounters.Insert > oc.Insert.Max {
oc.Insert.Max = ss.Opcounters.Insert oc.Insert.Max = delta.Opcounters.Insert
} }
if ss.Opcounters.Insert < oc.Insert.Min { if delta.Opcounters.Insert < oc.Insert.Min {
oc.Insert.Min = ss.Opcounters.Insert oc.Insert.Min = delta.Opcounters.Insert
} }
oc.Insert.Total += ss.Opcounters.Insert oc.Insert.Total += delta.Opcounters.Insert
// Query --------------------------------------- // Query ---------------------------------------
if ss.Opcounters.Query > oc.Query.Max { if delta.Opcounters.Query > oc.Query.Max {
oc.Query.Max = ss.Opcounters.Query oc.Query.Max = delta.Opcounters.Query
} }
if ss.Opcounters.Query < oc.Query.Min { if delta.Opcounters.Query < oc.Query.Min {
oc.Query.Min = ss.Opcounters.Query oc.Query.Min = delta.Opcounters.Query
} }
oc.Query.Total += ss.Opcounters.Query oc.Query.Total += delta.Opcounters.Query
// Command ------------------------------------- // Command -------------------------------------
if ss.Opcounters.Command > oc.Command.Max { if delta.Opcounters.Command > oc.Command.Max {
oc.Command.Max = ss.Opcounters.Command oc.Command.Max = delta.Opcounters.Command
} }
if ss.Opcounters.Command < oc.Command.Min { if delta.Opcounters.Command < oc.Command.Min {
oc.Command.Min = ss.Opcounters.Command oc.Command.Min = delta.Opcounters.Command
} }
oc.Command.Total += ss.Opcounters.Command oc.Command.Total += delta.Opcounters.Command
// Update -------------------------------------- // Update --------------------------------------
if ss.Opcounters.Update > oc.Update.Max { if delta.Opcounters.Update > oc.Update.Max {
oc.Update.Max = ss.Opcounters.Update oc.Update.Max = delta.Opcounters.Update
} }
if ss.Opcounters.Update < oc.Update.Min { if delta.Opcounters.Update < oc.Update.Min {
oc.Update.Min = ss.Opcounters.Update oc.Update.Min = delta.Opcounters.Update
} }
oc.Update.Total += ss.Opcounters.Update oc.Update.Total += delta.Opcounters.Update
// Delete -------------------------------------- // Delete --------------------------------------
if ss.Opcounters.Delete > oc.Delete.Max { if delta.Opcounters.Delete > oc.Delete.Max {
oc.Delete.Max = ss.Opcounters.Delete oc.Delete.Max = delta.Opcounters.Delete
} }
if ss.Opcounters.Delete < oc.Delete.Min { if delta.Opcounters.Delete < oc.Delete.Min {
oc.Delete.Min = ss.Opcounters.Delete oc.Delete.Min = delta.Opcounters.Delete
} }
oc.Delete.Total += ss.Opcounters.Delete oc.Delete.Total += delta.Opcounters.Delete
// GetMore ------------------------------------- // GetMore -------------------------------------
if ss.Opcounters.GetMore > oc.GetMore.Max { if delta.Opcounters.GetMore > oc.GetMore.Max {
oc.GetMore.Max = ss.Opcounters.GetMore oc.GetMore.Max = delta.Opcounters.GetMore
} }
if ss.Opcounters.GetMore < oc.GetMore.Min { if delta.Opcounters.GetMore < oc.GetMore.Min {
oc.GetMore.Min = ss.Opcounters.GetMore oc.GetMore.Min = delta.Opcounters.GetMore
} }
oc.GetMore.Total += ss.Opcounters.GetMore oc.GetMore.Total += delta.Opcounters.GetMore
previousoc.Insert.Total = ss.Opcounters.Insert prevOpCount.Insert.Total = ss.Opcounters.Insert
previousoc.Query.Total = ss.Opcounters.Query prevOpCount.Query.Total = ss.Opcounters.Query
previousoc.Command.Total = ss.Opcounters.Command prevOpCount.Command.Total = ss.Opcounters.Command
previousoc.Update.Total = ss.Opcounters.Update prevOpCount.Update.Total = ss.Opcounters.Update
previousoc.Delete.Total = ss.Opcounters.Delete prevOpCount.Delete.Total = ss.Opcounters.Delete
previousoc.GetMore.Total = ss.Opcounters.GetMore prevOpCount.GetMore.Total = ss.Opcounters.GetMore
} }
ticker.Stop() ticker.Stop()
oc.Insert.Avg = oc.Insert.Total / count oc.Insert.Avg = oc.Insert.Total
oc.Query.Avg = oc.Query.Total / count oc.Query.Avg = oc.Query.Total
oc.Update.Avg = oc.Update.Total / count oc.Update.Avg = oc.Update.Total
oc.Delete.Avg = oc.Delete.Total / count oc.Delete.Avg = oc.Delete.Total
oc.GetMore.Avg = oc.GetMore.Total / count oc.GetMore.Avg = oc.GetMore.Total
oc.Command.Avg = oc.Command.Total / count oc.Command.Avg = oc.Command.Total
// //
oc.SampleRate = time.Duration(count) * time.Second * sleep oc.SampleRate = time.Duration(count) * sleep
return oc, nil return oc, nil
} }

1
src/go/pt-mongodb-summary/templates/oplog.go
View File

@@ -6,5 +6,4 @@ Oplog Size {{.Size}} Mb
Oplog Used {{.UsedMB}} Mb Oplog Used {{.UsedMB}} Mb
Oplog Length {{.Running}} Oplog Length {{.Running}}
Last Election {{.ElectionTime}} Last Election {{.ElectionTime}}
` `

1
src/go/pt-mongodb-summary/templates/security.go
View File

@@ -6,5 +6,4 @@ Users {{.Users}}
Roles {{.Roles}} Roles {{.Roles}}
Auth {{.Auth}} Auth {{.Auth}}
SSL {{.SSL}} SSL {{.SSL}}
` `