daniel
/
distribution
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add notification metrics 

It adds notification related prometheus metrics, including:
  - total count for events/success/failure/error
  - total count for notification per each status code
  - gauge of the pending notification queue

Signed-off-by: tifayuki <tifayuki@gmail.com>
master
tifayuki 2018-02-13 13:30:56 -08:00 committed by Honglin Feng
parent 0d3efadf01
commit 8b70616846
2 changed files with 31 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
metrics/prometheus.go
View File

@ -10,4 +10,7 @@ const (
var ( var (
// StorageNamespace is the prometheus namespace of blob/cache related operations // StorageNamespace is the prometheus namespace of blob/cache related operations
StorageNamespace = metrics.NewNamespace(NamespacePrefix, "storage", nil) StorageNamespace = metrics.NewNamespace(NamespacePrefix, "storage", nil)
// NotificationsNamespace is the prometheus namespace of notification related metrics
NotificationsNamespace = metrics.NewNamespace(NamespacePrefix, "notifications", nil)
) )

28
notifications/metrics.go
View File

@ -5,6 +5,18 @@ import (
"fmt" "fmt"
"net/http" "net/http"
"sync" "sync"
prometheus "github.com/docker/distribution/metrics"
"github.com/docker/go-metrics"
)
var (
// eventsCounter counts total events of incoming, success, failure, and errors
eventsCounter = prometheus.NotificationsNamespace.NewLabeledCounter("events", "The number of total events", "type")
// pendingGauge measures the pending queue size
pendingGauge = prometheus.NotificationsNamespace.NewGauge("pending", "The gauge of pending events in queue", metrics.Total)
// statusCounter counts the total notification call per each status code
statusCounter = prometheus.NotificationsNamespace.NewLabeledCounter("status", "The number of status code", "code")
) )
// EndpointMetrics track various actions taken by the endpoint, typically by // EndpointMetrics track various actions taken by the endpoint, typically by
@ -61,6 +73,9 @@ func (emsl *endpointMetricsHTTPStatusListener) success(status int, events ...Eve
defer emsl.safeMetrics.Unlock() defer emsl.safeMetrics.Unlock()
emsl.Statuses[fmt.Sprintf("%d %s", status, http.StatusText(status))] += len(events) emsl.Statuses[fmt.Sprintf("%d %s", status, http.StatusText(status))] += len(events)
emsl.Successes += len(events) emsl.Successes += len(events)
statusCounter.WithValues(fmt.Sprintf("%d %s", status, http.StatusText(status))).Inc(1)
eventsCounter.WithValues("Successes").Inc(1)
} }
func (emsl *endpointMetricsHTTPStatusListener) failure(status int, events ...Event) { func (emsl *endpointMetricsHTTPStatusListener) failure(status int, events ...Event) {
@ -68,12 +83,17 @@ func (emsl *endpointMetricsHTTPStatusListener) failure(status int, events ...Eve
defer emsl.safeMetrics.Unlock() defer emsl.safeMetrics.Unlock()
emsl.Statuses[fmt.Sprintf("%d %s", status, http.StatusText(status))] += len(events) emsl.Statuses[fmt.Sprintf("%d %s", status, http.StatusText(status))] += len(events)
emsl.Failures += len(events) emsl.Failures += len(events)
statusCounter.WithValues(fmt.Sprintf("%d %s", status, http.StatusText(status))).Inc(1)
eventsCounter.WithValues("Failures").Inc(1)
} }
func (emsl *endpointMetricsHTTPStatusListener) err(err error, events ...Event) { func (emsl *endpointMetricsHTTPStatusListener) err(err error, events ...Event) {
emsl.safeMetrics.Lock() emsl.safeMetrics.Lock()
defer emsl.safeMetrics.Unlock() defer emsl.safeMetrics.Unlock()
emsl.Errors += len(events) emsl.Errors += len(events)
eventsCounter.WithValues("Errors").Inc(1)
} }
// endpointMetricsEventQueueListener maintains the incoming events counter and // endpointMetricsEventQueueListener maintains the incoming events counter and
@ -87,12 +107,17 @@ func (eqc *endpointMetricsEventQueueListener) ingress(events ...Event) {
defer eqc.Unlock() defer eqc.Unlock()
eqc.Events += len(events) eqc.Events += len(events)
eqc.Pending += len(events) eqc.Pending += len(events)
eventsCounter.WithValues("Events").Inc()
pendingGauge.Inc(1)
} }
func (eqc *endpointMetricsEventQueueListener) egress(events ...Event) { func (eqc *endpointMetricsEventQueueListener) egress(events ...Event) {
eqc.Lock() eqc.Lock()
defer eqc.Unlock() defer eqc.Unlock()
eqc.Pending -= len(events) eqc.Pending -= len(events)
pendingGauge.Dec(1)
} }
// endpoints is global registry of endpoints used to report metrics to expvar // endpoints is global registry of endpoints used to report metrics to expvar
@ -149,4 +174,7 @@ func init() {
})) }))
registry.(*expvar.Map).Set("notifications", &notifications) registry.(*expvar.Map).Set("notifications", &notifications)
// register prometheus metrics
metrics.Register(prometheus.NotificationsNamespace)
} }