ping-monitoring

A prometheus exporter to monitor matrix federation with using the existing pingbot infra.
git clone git://archive.git.mtrnord.blog/MTRNord/ping-monitoring.git
Log | Files | Refs | LICENSE

commit dcdf10dba5e2887040a8099caa15827a42fee7f1
parent 955ad1408a4951728e6a6e7b759864db472f6cd1
Author: MTRNord <mtrnord1@gmail.com>
Date:   Thu, 15 Feb 2024 20:29:07 +0100

Further fixes

Diffstat:
Mconfig.go | 18++++++++++--------
Mmain.go | 34++++++++++++++++++++++------------
Mping_manager.go | 52++++++++++++++++++++++++++++++++--------------------
3 files changed, 64 insertions(+), 40 deletions(-)

diff --git a/config.go b/config.go @@ -10,10 +10,12 @@ import ( ) type MatrixConfig struct { - Homeserver string `yaml:"homeserver"` - Username string `yaml:"username"` - Password string `yaml:"password"` - Client *mautrix.Client + Homeserver string `yaml:"homeserver"` + Username string `yaml:"username"` + Password string `yaml:"password,omitempty"` + DeviceID string `yaml:"device_id,omitempty"` + AccessToken string `yaml:"access_token,omitempty"` + Client *mautrix.Client } type Config struct { @@ -74,8 +76,8 @@ func ReadConfig() Config { log.Errorln("No own homeserver username defined") os.Exit(1) } - if config.OwnHomeserver.Password == "" { - log.Errorln("No own homeserver password defined") + if config.OwnHomeserver.Password == "" && config.OwnHomeserver.AccessToken == "" && config.OwnHomeserver.DeviceID == "" { + log.Errorln("No own homeserver password or access token defined") os.Exit(1) } @@ -95,8 +97,8 @@ func ReadConfig() Config { log.Errorf("No remote homeserver username defined for homeserver %s", remoteHomeserver.Homeserver) os.Exit(1) } - if remoteHomeserver.Password == "" { - log.Errorf("No remote homeserver password defined for homeserver %s", remoteHomeserver.Homeserver) + if remoteHomeserver.AccessToken == "" && remoteHomeserver.DeviceID == "" && remoteHomeserver.Password == "" { + log.Errorf("No remote homeserver password or access token defined for homeserver %s", remoteHomeserver.Homeserver) os.Exit(1) } } diff --git a/main.go b/main.go @@ -72,18 +72,28 @@ func createMatrixClient(config *Config, homeserver *MatrixConfig) *mautrix.Clien ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - loginresp, err := client.Login(ctx, &mautrix.ReqLogin{ - Type: mautrix.AuthTypePassword, - Identifier: mautrix.UserIdentifier{Type: "m.id.user", User: homeserver.Username}, - Password: homeserver.Password, - StoreCredentials: true, - StoreHomeserverURL: true, - }) - if err != nil { - log.Errorf("Failed to login to %s: %s", homeserver.Homeserver, err) - os.Exit(1) + if homeserver.AccessToken == "" { + loginresp, err := client.Login(ctx, &mautrix.ReqLogin{ + Type: mautrix.AuthTypePassword, + Identifier: mautrix.UserIdentifier{Type: "m.id.user", User: homeserver.Username}, + Password: homeserver.Password, + StoreCredentials: true, + StoreHomeserverURL: true, + }) + if err != nil { + log.Errorf("Failed to login to %s: %s", homeserver.Homeserver, err) + os.Exit(1) + } + log.Infof("Logged in as %s", loginresp.UserID) + + // Tell user to store the access token and device id and remove the password from the config + log.Warnf("Please store the access token and device id for %s and remove the password from the config", homeserver.Homeserver) + log.Warnf("Access token: %s", loginresp.AccessToken) + log.Warnf("Device ID: %s", loginresp.DeviceID) + } else { + client.AccessToken = homeserver.AccessToken + client.DeviceID = id.DeviceID(homeserver.DeviceID) } - log.Infof("Logged in as %s", loginresp.UserID) // join the ping room resp, err := client.JoinRoom(ctx, config.PingRoom, "", nil) @@ -108,7 +118,7 @@ func createMatrixClient(config *Config, homeserver *MatrixConfig) *mautrix.Clien } if evt.Content.AsMessage().Body == "!ping" { - log.Infoln("Received ping message in pingroom by ", evt.Sender) + log.Infoln("Received ping message in pingroom by", evt.Sender) // Respond to the ping pong_event_content := PongEventContent{ MsgType: "m.notice", diff --git a/ping_manager.go b/ping_manager.go @@ -16,14 +16,14 @@ import ( type PingCollector struct { Config *Config Metrics map[string]*prometheus.Desc - LastCollected time.Time + LastCollected map[string]time.Time Failures map[string]int } func (c *PingCollector) Describe(ch chan<- *prometheus.Desc) { - mean := prometheus.NewDesc(prometheus.BuildFQName(collector, "", "mean"), "Mean ping time", []string{"homeserver"}, nil) - median := prometheus.NewDesc(prometheus.BuildFQName(collector, "", "median"), "Median ping time", []string{"homeserver"}, nil) - gmean := prometheus.NewDesc(prometheus.BuildFQName(collector, "", "gmean"), "GMean ping time", []string{"homeserver"}, nil) + mean := prometheus.NewDesc(prometheus.BuildFQName(collector, "", "mean"), "Mean ping time", []string{"homeserver", "origin", "direction"}, nil) + median := prometheus.NewDesc(prometheus.BuildFQName(collector, "", "median"), "Median ping time", []string{"homeserver", "origin", "direction"}, nil) + gmean := prometheus.NewDesc(prometheus.BuildFQName(collector, "", "gmean"), "GMean ping time", []string{"homeserver", "origin", "direction"}, nil) failures := prometheus.NewDesc(prometheus.BuildFQName(collector, "", "failures"), "Ping failures", []string{"origin", "direction"}, nil) c.Metrics = make(map[string]*prometheus.Desc) c.Metrics["mean"] = mean @@ -61,10 +61,14 @@ func (c *PingCollector) Collect(ch chan<- prometheus.Metric) { // It takes a homeserver config as an argument to know where to send the ping func (c *PingCollector) SendPing(ctx context.Context, client *mautrix.Client, ch chan<- prometheus.Metric, wg *sync.WaitGroup) { defer wg.Done() - if c.LastCollected.Add(time.Duration(c.Config.PingRateSeconds) * time.Second).After(time.Now()) { + if c.LastCollected[client.UserID.Homeserver()].Add(time.Duration(c.Config.PingRateSeconds) * time.Second).After(time.Now()) { log.Infof("Not sending ping as we sent one less than %d seconds ago", c.Config.PingRateSeconds) return } + if c.LastCollected == nil { + c.LastCollected = make(map[string]time.Time) + } + c.LastCollected[client.UserID.Homeserver()] = time.Now() log.Infof("Sending ping as %s", client.UserID) if c.Config.PingRoomID == "" { log.Errorf("No ping room ID found") @@ -82,9 +86,15 @@ func (c *PingCollector) SendPing(ctx context.Context, client *mautrix.Client, ch // Otherwise timeout after PingThresholdSeconds var currentData Data var gotKnownPong bool = false + var pingTime time.Time = time.Now() + u, err := url.Parse(c.Config.OwnHomeserver.Homeserver) + if err != nil { + log.Errorf("Failed to parse homeserver url: %s", err) + return + } outer: - for time.Now().Before(c.LastCollected.Add(time.Duration(c.Config.PingThresholdSeconds) * time.Second)) { + for time.Now().Before(pingTime.Add(time.Duration(c.Config.PingThresholdSeconds) * time.Second)) { resp, err := http.Get(c.Config.PingJsonURL) if err != nil { log.Errorf("Failed to get ping json: %s", err) @@ -97,12 +107,13 @@ outer: json.NewDecoder(resp.Body).Decode(&currentData) // Check if we have a pong for this ping - if ping, ok := currentData.Pings[client.HomeserverURL.Host]; ok { + log.Infof("Checking for pong for ping as %s", client.UserID.Homeserver()) + if ping, ok := currentData.Pings[client.UserID.Homeserver()]; ok { log.Infof("Got pong for ping as %s", client.UserID) // Check if its from a known remote homeserver in ping.Pongs for homeserver := range ping.Pongs { - if homeserver == client.HomeserverURL.Host { + if homeserver == client.UserID.Homeserver() { continue } if _, ok := currentData.Pings[homeserver]; ok { @@ -126,27 +137,28 @@ outer: if c.Failures == nil { c.Failures = make(map[string]int) } - u, err := url.Parse(c.Config.OwnHomeserver.Homeserver) - if err != nil { - log.Errorf("Failed to parse homeserver url: %s", err) - return - } if client.HomeserverURL.Host == u.Host { c.Failures["outgoing"]++ - ch <- prometheus.MustNewConstMetric(c.Metrics["failures"], prometheus.CounterValue, float64(c.Failures["outgoing"]), client.HomeserverURL.Host, "outgoing") + ch <- prometheus.MustNewConstMetric(c.Metrics["failures"], prometheus.CounterValue, float64(c.Failures["outgoing"]), client.UserID.Homeserver(), "outgoing") } else { c.Failures["incoming"]++ - ch <- prometheus.MustNewConstMetric(c.Metrics["failures"], prometheus.CounterValue, float64(c.Failures["incoming"]), client.HomeserverURL.Host, "incoming") + ch <- prometheus.MustNewConstMetric(c.Metrics["failures"], prometheus.CounterValue, float64(c.Failures["incoming"]), client.UserID.Homeserver(), "incoming") } } // Update mean, median and gmean metrics // All of these are per homeserver we received a pong from - // They are all of type histogram - for homeserver, ping := range currentData.Pings[client.HomeserverURL.Host].Pongs { - ch <- prometheus.MustNewConstMetric(c.Metrics["mean"], prometheus.GaugeValue, ping.Mean, homeserver) - ch <- prometheus.MustNewConstMetric(c.Metrics["median"], prometheus.GaugeValue, ping.Median, homeserver) - ch <- prometheus.MustNewConstMetric(c.Metrics["gmean"], prometheus.GaugeValue, ping.GMean, homeserver) + // They are all of type Gauge (Should they be a historgram?) + for homeserver, ping := range currentData.Pings[client.UserID.Homeserver()].Pongs { + if client.HomeserverURL.Host == u.Host { + ch <- prometheus.MustNewConstMetric(c.Metrics["mean"], prometheus.GaugeValue, ping.Mean, homeserver, client.UserID.Homeserver(), "outgoing") + ch <- prometheus.MustNewConstMetric(c.Metrics["median"], prometheus.GaugeValue, ping.Median, homeserver, client.UserID.Homeserver(), "outgoing") + ch <- prometheus.MustNewConstMetric(c.Metrics["gmean"], prometheus.GaugeValue, ping.GMean, homeserver, client.UserID.Homeserver(), "outgoing") + } else { + ch <- prometheus.MustNewConstMetric(c.Metrics["mean"], prometheus.GaugeValue, ping.Mean, homeserver, client.UserID.Homeserver(), "incoming") + ch <- prometheus.MustNewConstMetric(c.Metrics["median"], prometheus.GaugeValue, ping.Median, homeserver, client.UserID.Homeserver(), "incoming") + ch <- prometheus.MustNewConstMetric(c.Metrics["gmean"], prometheus.GaugeValue, ping.GMean, homeserver, client.UserID.Homeserver(), "incoming") + } } }