WIP

src/go/mongolib/profiler/profiler.go

@@ -15,6 +15,7 @@ import (
 
 var (
 	MAX_DEPTH_LEVEL        = 10
+	DOCS_BUFFER_SIZE       = 100
 	CANNOT_GET_QUERY_ERROR = errors.New("cannot get query field from the profile document (it is not a map)")
 )
 
@@ -58,6 +59,7 @@ type Iter interface {
 }
 
 type Profiler interface {
+	GetLastError() error
 	StatsChan() chan []Stat
 	Start()
 	Stop()
@@ -69,10 +71,12 @@ type Profile struct {
 	ticker        chan time.Time
 	statsChan     chan []Stat
 	stopChan      chan bool
-	stats         []Stat
+	docsChan      chan proto.SystemProfile
+	rawStats      map[StatsGroupKey]*Stat
 	keyFilters    []string
 	fingerprinter fingerprinter.Fingerprinter
 	running       bool
+	lastError     error
 }
 
 func NewProfiler(iterator Iter, filters []filter.Filter, ticker chan time.Time, fp fingerprinter.Fingerprinter) Profiler {
@@ -82,11 +86,16 @@ func NewProfiler(iterator Iter, filters []filter.Filter, ticker chan time.Time,
 		iterator:      iterator,
 		ticker:        ticker,
 		statsChan:     make(chan []Stat),
-		stats:         make([]Stat, 100),
+		docsChan:      make(chan proto.SystemProfile, DOCS_BUFFER_SIZE),
+		rawStats:      make(map[StatsGroupKey]*Stat),
 		keyFilters:    []string{"^shardVersion$", "^\\$"},
 	}
 }
 
+func (p *Profile) GetLastError() error {
+	return p.lastError
+}
+
 func (p *Profile) StatsChan() chan []Stat {
 	return p.statsChan
 }
@@ -100,16 +109,33 @@ func (p *Profile) Start() {
 
 func (p *Profile) Stop() {
 	if p.running {
-		p.stopChan <- true
+		p.iterator.Close()
+		close(p.stopChan)
 	}
 }
 
 func (p *Profile) getData() {
-	var doc proto.SystemProfile
+	go p.getDocs()
-	stop := false
+MAIN_GETDATA_LOOP:
-	stats := make(map[StatsGroupKey]*Stat)
+	for {
+		select {
+		case <-p.ticker:
+			p.statsChan <- statsToArray(p.rawStats)
+			p.rawStats = make(map[StatsGroupKey]*Stat) // Reset stats
+		case <-p.stopChan:
+			p.iterator.Close()
+			break MAIN_GETDATA_LOOP
+		}
+	}
+}
 
-	for !stop && p.iterator.Next(&doc) && p.iterator.Err() == nil {
+func (p *Profile) getDocs() {
+	var doc proto.SystemProfile
+
+	for p.iterator.Next(&doc) || p.iterator.Timeout() {
+		if p.iterator.Timeout() {
+			continue
+		}
 		valid := true
 		for _, filter := range p.filters {
 			if filter(doc) == false {
@@ -120,58 +146,49 @@ func (p *Profile) getData() {
 		if !valid {
 			continue
 		}
+		if len(doc.Query) > 0 {
 
-		select {
+			fp, err := p.fingerprinter.Fingerprint(doc.Query)
-		case <-p.ticker:
+			if err != nil {
-			p.statsChan <- statsToArray(stats)
+				log.Errorf("cannot get fingerprint: %s", err.Error())
-		case <-p.stopChan:
+				continue
-			stop = true
+			}
-			continue
+			var s *Stat
-		default:
+			var ok bool
-			if len(doc.Query) > 0 {
+			key := StatsGroupKey{
-
+				Operation:   doc.Op,
-				fp, err := p.fingerprinter.Fingerprint(doc.Query)
+				Fingerprint: fp,
-				if err != nil {
+				Namespace:   doc.Ns,
-					log.Errorf("cannot get fingerprint: %s", err.Error())
+			}
-					continue
+			if s, ok = p.rawStats[key]; !ok {
-				}
+				realQuery, _ := util.GetQueryField(doc.Query)
-				var s *Stat
+				s = &Stat{
-				var ok bool
+					ID:          fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("%s", key)))),
-				key := StatsGroupKey{
 					Operation:   doc.Op,
 					Fingerprint: fp,
 					Namespace:   doc.Ns,
+					TableScan:   false,
+					Query:       realQuery,
 				}
-				if s, ok = stats[key]; !ok {
+				p.rawStats[key] = s
-					realQuery, _ := util.GetQueryField(doc.Query)
+			}
-					s = &Stat{
+			s.Count++
-						ID:          fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("%s", key)))),
+			s.NScanned = append(s.NScanned, float64(doc.DocsExamined))
-						Operation:   doc.Op,
+			s.NReturned = append(s.NReturned, float64(doc.Nreturned))
-						Fingerprint: fp,
+			s.QueryTime = append(s.QueryTime, float64(doc.Millis))
-						Namespace:   doc.Ns,
+			s.ResponseLength = append(s.ResponseLength, float64(doc.ResponseLength))
-						TableScan:   false,
+			var zeroTime time.Time
-						Query:       realQuery,
+			if s.FirstSeen == zeroTime || s.FirstSeen.After(doc.Ts) {
-					}
+				s.FirstSeen = doc.Ts
-					stats[key] = s
+			}
-				}
+			if s.LastSeen == zeroTime || s.LastSeen.Before(doc.Ts) {
-				s.Count++
+				s.LastSeen = doc.Ts
-				s.NScanned = append(s.NScanned, float64(doc.DocsExamined))
-				s.NReturned = append(s.NReturned, float64(doc.Nreturned))
-				s.QueryTime = append(s.QueryTime, float64(doc.Millis))
-				s.ResponseLength = append(s.ResponseLength, float64(doc.ResponseLength))
-				var zeroTime time.Time
-				if s.FirstSeen == zeroTime || s.FirstSeen.After(doc.Ts) {
-					s.FirstSeen = doc.Ts
-				}
-				if s.LastSeen == zeroTime || s.LastSeen.Before(doc.Ts) {
-					s.LastSeen = doc.Ts
-				}
 			}
 		}
 	}
-
+	p.statsChan <- statsToArray(p.rawStats)
-	p.statsChan <- statsToArray(stats)
 	p.running = false
+	p.stopChan <- true
 }
 
 func statsToArray(stats map[StatsGroupKey]*Stat) []Stat {