From 01276418c57725bb167bb1883c4b51fc458862e3 Mon Sep 17 00:00:00 2001 From: Christian Weichel Date: Wed, 9 Dec 2020 08:46:57 +0000 Subject: [PATCH] [ws-scheduler] Add Prometheus metrics to scaler --- components/ee/ws-scheduler/cmd/run.go | 9 ++- components/ee/ws-scheduler/go.mod | 8 +-- components/ee/ws-scheduler/go.sum | 26 ++++++++ .../ee/ws-scheduler/pkg/scaler/driver.go | 12 +++- .../ee/ws-scheduler/pkg/scaler/metrics.go | 59 +++++++++++++++++++ 5 files changed, 107 insertions(+), 7 deletions(-) create mode 100644 components/ee/ws-scheduler/pkg/scaler/metrics.go diff --git a/components/ee/ws-scheduler/cmd/run.go b/components/ee/ws-scheduler/cmd/run.go index dd4f08849c..610f2f8a60 100644 --- a/components/ee/ws-scheduler/cmd/run.go +++ b/components/ee/ws-scheduler/cmd/run.go @@ -58,6 +58,8 @@ var runCmd = &cobra.Command{ log.Info("ws-scheduler shut down") }() + reg := prometheus.NewRegistry() + if config.Scaler.Enabled { controller, err := scaler.NewController(config.Scaler.Controller) if err != nil { @@ -67,14 +69,17 @@ var runCmd = &cobra.Command{ if err != nil { log.WithError(err).Fatal("cannot create scaler driver") } + err = driver.RegisterMetrics(prometheus.WrapRegistererWithPrefix("gitpod_ws_scaler_", reg)) + if err != nil { + log.WithError(err).Fatal("cannot register metrics") + } + go driver.Run() defer driver.Stop() - log.WithField("controller", config.Scaler.Controller.Kind).Info("started scaler") } if config.Prometheus.Addr != "" { - reg := prometheus.NewRegistry() reg.MustRegister( prometheus.NewGoCollector(), prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}), diff --git a/components/ee/ws-scheduler/go.mod b/components/ee/ws-scheduler/go.mod index 5c94dc1f05..38810a5c29 100644 --- a/components/ee/ws-scheduler/go.mod +++ b/components/ee/ws-scheduler/go.mod @@ -11,9 +11,9 @@ require ( github.com/go-ozzo/ozzo-validation v3.5.0+incompatible github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 // indirect github.com/golang/mock v1.4.3 - github.com/golang/protobuf v1.3.5 - github.com/google/go-cmp v0.4.0 - github.com/google/uuid v1.1.1 + github.com/golang/protobuf v1.4.2 + github.com/google/go-cmp v0.5.0 + github.com/google/uuid v1.1.2 github.com/googleapis/gnostic v0.3.1 // indirect github.com/hashicorp/golang-lru v0.5.3 // indirect github.com/opentracing/opentracing-go v1.1.0 @@ -23,7 +23,7 @@ require ( golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 google.golang.org/appengine v1.6.5 // indirect - google.golang.org/grpc v1.29.1 + google.golang.org/grpc v1.34.0 k8s.io/api v0.0.0 k8s.io/apimachinery v0.0.0 k8s.io/client-go v0.0.0 diff --git a/components/ee/ws-scheduler/go.sum b/components/ee/ws-scheduler/go.sum index 9759e80a8b..2c9576d822 100644 --- a/components/ee/ws-scheduler/go.sum +++ b/components/ee/ws-scheduler/go.sum @@ -55,6 +55,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cfssl v0.0.0-20180726162950-56268a613adf/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA= github.com/clusterhq/flocker-go v0.0.0-20160920122132-2b8b7259d313/go.mod h1:P1wt9Z3DP8O6W3rvwCt0REIlshg1InHImaLW0t3ObY0= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codedellemc/goscaleio v0.0.0-20170830184815-20e2ce2cf885/go.mod h1:JIHmDHNZO4tmA3y3RHp6+Gap6kFsNf55W9Pn/3YS9IY= @@ -95,7 +96,9 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550 h1:mV9jbLoSW/8m4VK16ZkHTozJa8sesK5u5kTMFysTYac= @@ -161,6 +164,13 @@ github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8= github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= @@ -172,8 +182,10 @@ github.com/google/cadvisor v0.33.2-0.20190411163913-9db8c7dee20a/go.mod h1:1nql6 github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= @@ -185,6 +197,7 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= @@ -369,6 +382,7 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/syndtr/gocapability v0.0.0-20160928074757-e7cb7fa329f4/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= @@ -543,6 +557,7 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.13.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -550,8 +565,19 @@ google.golang.org/grpc v1.21.1 h1:j6XxA85m/6txkUCHvzlV5f+HBNl/1r5cZ2A/3IEFOO8= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/components/ee/ws-scheduler/pkg/scaler/driver.go b/components/ee/ws-scheduler/pkg/scaler/driver.go index 049e908109..7f68f916b4 100644 --- a/components/ee/ws-scheduler/pkg/scaler/driver.go +++ b/components/ee/ws-scheduler/pkg/scaler/driver.go @@ -16,6 +16,7 @@ import ( "github.com/gitpod-io/gitpod/ws-manager/api" "github.com/golang/protobuf/ptypes" "github.com/google/uuid" + "github.com/prometheus/client_golang/prometheus" "golang.org/x/xerrors" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -80,7 +81,8 @@ type WorkspaceManagerPrescaleDriver struct { Controller Controller - time timer + time timer + metrics *metrics stop chan struct{} once sync.Once @@ -91,6 +93,12 @@ type workspaceStatus struct { DeletionCandidates []string } +// RegisterMetrics registers prometheus metrics for this driver +func (wspd *WorkspaceManagerPrescaleDriver) RegisterMetrics(reg prometheus.Registerer) error { + wspd.metrics = newMetrics() + return wspd.metrics.Register(reg) +} + // Run runs the prescale driver until Stop() is called func (wspd *WorkspaceManagerPrescaleDriver) Run() { ctx, cancel := context.WithCancel(context.Background()) @@ -141,6 +149,7 @@ func (wspd *WorkspaceManagerPrescaleDriver) Run() { } status.Count.Ghost += len(startingGhosts) counts <- status.Count + wspd.metrics.OnGhostCountChange(status.Count.Ghost) log.WithField("counts", status.Count).Debug("status update") case <-houseKeeping: for id, t1 := range startingGhosts { @@ -179,6 +188,7 @@ func (wspd *WorkspaceManagerPrescaleDriver) Run() { log.WithField("delta", d).Info("renewed ghost workspaces") case setpoint = <-cchan: // we've already set the new setpoint - wait for scheduleGhosts to act on it. + wspd.metrics.OnSetpointChange(setpoint) case <-scheduleGhosts: d := setpoint - status.Count.Ghost if d == 0 { diff --git a/components/ee/ws-scheduler/pkg/scaler/metrics.go b/components/ee/ws-scheduler/pkg/scaler/metrics.go new file mode 100644 index 0000000000..62f70dc859 --- /dev/null +++ b/components/ee/ws-scheduler/pkg/scaler/metrics.go @@ -0,0 +1,59 @@ +// Copyright (c) 2020 TypeFox GmbH. All rights reserved. +// Licensed under the Gitpod Enterprise Source Code License, +// See License.enterprise.txt in the project root folder. + +package scaler + +import "github.com/prometheus/client_golang/prometheus" + +type metrics struct { + setpointGauge prometheus.Gauge + ghostCountGauge prometheus.Gauge +} + +func newMetrics() *metrics { + return &metrics{ + setpointGauge: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "driver_setpoint", + Help: "setpoint of the scaler driver", + }), + ghostCountGauge: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "ghost_count", + Help: "current number of Ghost workspaces scaler is aware of", + }), + } +} + +// Register registers all metrics scaler can export +func (m *metrics) Register(reg prometheus.Registerer) error { + if m == nil { + return nil + } + + collectors := []prometheus.Collector{ + m.setpointGauge, + m.ghostCountGauge, + } + for _, c := range collectors { + err := reg.Register(c) + if err != nil { + return err + } + } + + return nil +} + +func (m *metrics) OnSetpointChange(v int) { + if m == nil { + return + } + m.setpointGauge.Set(float64(v)) +} + +func (m *metrics) OnGhostCountChange(v int) { + if m == nil { + return + } + m.ghostCountGauge.Set(float64(v)) +}