ping-monitoring

A prometheus exporter to monitor matrix federation with using the existing pingbot infra.
git clone git://archive.git.mtrnord.blog/MTRNord/ping-monitoring.git
Log | Files | Refs | LICENSE

commit 955ad1408a4951728e6a6e7b759864db472f6cd1
parent 417f472ab0c38651bb193a0f0848e6213d3379cd
Author: MTRNord <mtrnord1@gmail.com>
Date:   Thu, 15 Feb 2024 20:00:45 +0100

Various small fixes

Diffstat:
ADockerfile | 22++++++++++++++++++++++
Mconfig.go | 6+++---
Mmain.go | 53+++++++++++++++++++++++++++--------------------------
Mping_manager.go | 40+++++++++++++++++++++++++++++++++++++---
4 files changed, 89 insertions(+), 32 deletions(-)

diff --git a/Dockerfile b/Dockerfile @@ -0,0 +1,22 @@ +FROM golang:1.22 AS build-stage + +WORKDIR /app + +COPY go.mod go.sum ./ +RUN go mod download + +COPY *.go ./ + +RUN CGO_ENABLED=0 GOOS=linux go build -o /ping-monitoring + +FROM gcr.io/distroless/base-debian12 + +WORKDIR / + +COPY --from=build-stage /ping-monitoring /ping-monitoring + +EXPOSE 8080 + +USER nonroot:nonroot + +ENTRYPOINT ["/ping-monitoring"] diff --git a/config.go b/config.go @@ -112,13 +112,13 @@ func ReadConfig() Config { } // Ensure that the Remote Homeservers are different from the Own Homeserver or each other - for _, remoteHomeserver := range config.RemoteHomeservers { + for i, remoteHomeserver := range config.RemoteHomeservers { if remoteHomeserver.Homeserver == config.OwnHomeserver.Homeserver { log.Errorf("Remote homeserver %s is the same as the own homeserver", remoteHomeserver.Homeserver) os.Exit(1) } - for _, remoteHomeserver2 := range config.RemoteHomeservers { - if remoteHomeserver.Homeserver == remoteHomeserver2.Homeserver { + for j, remoteHomeserver2 := range config.RemoteHomeservers { + if remoteHomeserver.Homeserver == remoteHomeserver2.Homeserver && i != j { log.Errorf("Remote homeserver %s is the same as remote homeserver %s", remoteHomeserver.Homeserver, remoteHomeserver2.Homeserver) os.Exit(1) } diff --git a/main.go b/main.go @@ -28,22 +28,21 @@ func main() { flag.Parse() // Create a matrix client for each homeserver - config.OwnHomeserver.Client = createMatrixClient(config, config.OwnHomeserver) + config.OwnHomeserver.Client = createMatrixClient(&config, &config.OwnHomeserver) for i := range config.RemoteHomeservers { - config.RemoteHomeservers[i].Client = createMatrixClient(config, config.RemoteHomeservers[i]) + config.RemoteHomeservers[i].Client = createMatrixClient(&config, &config.RemoteHomeservers[i]) } - prometheus.Register(version.NewCollector(collector)) - prometheus.Register(&PingCollector{ - Config: config, + reg := prometheus.NewRegistry() + reg.MustRegister(version.NewCollector(collector)) + reg.MustRegister(&PingCollector{ + Config: &config, }) - http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) { - h := promhttp.HandlerFor(prometheus.Gatherers{ - prometheus.DefaultGatherer, - }, promhttp.HandlerOpts{}) - h.ServeHTTP(w, r) - }) + http.Handle("/metrics", promhttp.HandlerFor( + reg, + promhttp.HandlerOpts{}, + )) log.Infof("Starting http server - %s", address) if err := http.ListenAndServe(address, nil); err != nil { @@ -62,38 +61,42 @@ type PongEventContent struct { RelatesTo PongEventRelation `json:"m.relates_to"` } -func createMatrixClient(Config Config, Homeserver MatrixConfig) *mautrix.Client { +func createMatrixClient(config *Config, homeserver *MatrixConfig) *mautrix.Client { var startTime = time.Now() - client, err := mautrix.NewClient(Homeserver.Homeserver, id.UserID(Homeserver.Username), "") + client, err := mautrix.NewClient(homeserver.Homeserver, id.UserID(homeserver.Username), "") if err != nil { - log.Errorf("Failed to create client for %s: %s", Homeserver.Homeserver, err) + log.Errorf("Failed to create client for %s: %s", homeserver.Homeserver, err) os.Exit(1) } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - _, err = client.Login(ctx, &mautrix.ReqLogin{ - Type: mautrix.AuthTypePassword, - Identifier: mautrix.UserIdentifier{Type: "m.id.user", User: Homeserver.Username}, - Password: Homeserver.Password, + loginresp, err := client.Login(ctx, &mautrix.ReqLogin{ + Type: mautrix.AuthTypePassword, + Identifier: mautrix.UserIdentifier{Type: "m.id.user", User: homeserver.Username}, + Password: homeserver.Password, + StoreCredentials: true, + StoreHomeserverURL: true, }) if err != nil { - log.Errorf("Failed to login to %s: %s", Homeserver.Homeserver, err) + log.Errorf("Failed to login to %s: %s", homeserver.Homeserver, err) os.Exit(1) } + log.Infof("Logged in as %s", loginresp.UserID) // join the ping room - resp, err := client.JoinRoom(ctx, Config.PingRoom, "", nil) + resp, err := client.JoinRoom(ctx, config.PingRoom, "", nil) if err != nil { log.Errorf("Failed to join ping room: %s", err) os.Exit(1) } - Config.PingRoomID = resp.RoomID + log.Infof("Joined ping room %s", resp.RoomID) + config.PingRoomID = resp.RoomID syncer := client.Syncer.(*mautrix.DefaultSyncer) syncer.OnEventType(event.EventMessage, func(ctx context.Context, evt *event.Event) { - if evt.RoomID != Config.PingRoomID { + if evt.RoomID != config.PingRoomID { return } if evt.Sender == client.UserID { @@ -105,6 +108,7 @@ func createMatrixClient(Config Config, Homeserver MatrixConfig) *mautrix.Client } if evt.Content.AsMessage().Body == "!ping" { + log.Infoln("Received ping message in pingroom by ", evt.Sender) // Respond to the ping pong_event_content := PongEventContent{ MsgType: "m.notice", @@ -121,11 +125,8 @@ func createMatrixClient(Config Config, Homeserver MatrixConfig) *mautrix.Client } }) - syncCtx, cancelSync := context.WithCancel(context.Background()) - defer cancelSync() - go func() { - err = client.SyncWithContext(syncCtx) + err = client.Sync() if err != nil && !errors.Is(err, context.Canceled) { log.Errorf("Failed to sync: %s", err) os.Exit(1) diff --git a/ping_manager.go b/ping_manager.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "net/http" + "net/url" + "sync" "time" "github.com/prometheus/client_golang/prometheus" @@ -12,7 +14,7 @@ import ( ) type PingCollector struct { - Config Config + Config *Config Metrics map[string]*prometheus.Desc LastCollected time.Time Failures map[string]int @@ -40,16 +42,34 @@ func (c *PingCollector) Describe(ch chan<- *prometheus.Desc) { // We also do react on our own `!ping` message and send a `!pong` back to the ping room based on the maubot echobot logic. func (c *PingCollector) Collect(ch chan<- prometheus.Metric) { log.Infoln("Starting Collecting metrics") + var wg sync.WaitGroup + wg.Add(1) + + // Send ping from our own homeserver + go c.SendPing(context.Background(), c.Config.OwnHomeserver.Client, ch, &wg) + + // Send ping from all remote homeservers + for _, homeserver := range c.Config.RemoteHomeservers { + wg.Add(1) + go c.SendPing(context.Background(), homeserver.Client, ch, &wg) + } + + wg.Wait() } // This sends a ping into the ping room // It takes a homeserver config as an argument to know where to send the ping -func (c *PingCollector) SendPing(ctx context.Context, client mautrix.Client, ch chan<- prometheus.Metric) { +func (c *PingCollector) SendPing(ctx context.Context, client *mautrix.Client, ch chan<- prometheus.Metric, wg *sync.WaitGroup) { + defer wg.Done() if c.LastCollected.Add(time.Duration(c.Config.PingRateSeconds) * time.Second).After(time.Now()) { log.Infof("Not sending ping as we sent one less than %d seconds ago", c.Config.PingRateSeconds) return } log.Infof("Sending ping as %s", client.UserID) + if c.Config.PingRoomID == "" { + log.Errorf("No ping room ID found") + return + } // Send the ping resp, err := client.SendText(ctx, c.Config.PingRoomID, "!ping") @@ -92,13 +112,27 @@ outer: break outer } } + } else { + log.Warnf("No pong for ping as %s", client.UserID) } + + // Sleep for 1 second + time.Sleep(1 * time.Second) } // If we have not received a pong for this ping, we should log it if !gotKnownPong { log.Errorf("Failed to get pong for ping as %s", client.UserID) - if client.HomeserverURL.Host == c.Config.OwnHomeserver.Homeserver { + if c.Failures == nil { + c.Failures = make(map[string]int) + } + u, err := url.Parse(c.Config.OwnHomeserver.Homeserver) + if err != nil { + log.Errorf("Failed to parse homeserver url: %s", err) + return + } + + if client.HomeserverURL.Host == u.Host { c.Failures["outgoing"]++ ch <- prometheus.MustNewConstMetric(c.Metrics["failures"], prometheus.CounterValue, float64(c.Failures["outgoing"]), client.HomeserverURL.Host, "outgoing") } else {