Move metrics to metrics/ pacakge

docs/config.md

@@ -2159,13 +2159,14 @@ See [Installation for Docker](install.md#docker) for an example of how this coul
 If configured, ntfy can expose a `/metrics` endpoint for [Prometheus](https://prometheus.io/), which can then be used to
 create dashboards and alerts (e.g. via [Grafana](https://grafana.com/)).
 
-To configure the metrics endpoint, either set `enable-metrics` and/or set the `metrics-listen-http` option to a dedicated
+To configure the metrics endpoint, either set `enable-metrics`, or set the `metrics-listen-http` option to a dedicated
 listen address. Metrics may be considered sensitive information, so before you enable them, be sure you know what you are
 doing, and/or secure access to the endpoint in your reverse proxy.
 
 - `enable-metrics` enables the /metrics endpoint for the default ntfy server (i.e. HTTP, HTTPS and/or Unix socket)
-- `metrics-listen-http` exposes the metrics endpoint via a dedicated `[IP]:port`. If set, this option implicitly
+- `metrics-listen-http` moves the metrics endpoint to a dedicated `[IP]:port`, e.g. "10.0.1.1:9090" or ":9090". It
-  enables metrics as well, e.g. "10.0.1.1:9090" or ":9090"
+  implicitly enables metrics. If set, the metrics are served only on that dedicated port, and the default ntfy server
+  does not serve /metrics, even if `enable-metrics` is also set.
 
 === "server.yml (Using default port)"
     ```yaml

docs/releases.md

@@ -2016,6 +2016,7 @@ and the [ntfy Android app](https://github.com/binwiederhier/ntfy-android/release
 **Bug fixes + maintenance:**
 
 * Fix Twilio phone calls and phone number verifications failing silently when Twilio rejected the request, and move the Twilio integration into its own `twilio` package
+* Move the Prometheus metrics into a dedicated `metrics` package
 
 ### ntfy Android v1.25.2 (UNRELEASED)
 

metrics/metrics.go

@@ -0,0 +1,107 @@
+// Package metrics defines the Prometheus metrics exposed by the ntfy server, and registers them
+// with the default Prometheus registry on import. It is decoupled from the ntfy server, so that
+// call sites can update metrics without depending on the server package.
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Collectors for all metrics exposed by the server.
+//
+// These are never nil, so that call sites can update them unconditionally. If metrics are
+// disabled, the server never mounts the /metrics handler, and the values are simply never read.
+var (
+	MessagesPublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "ntfy_messages_published_success",
+	})
+	MessagesPublishedFailure = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "ntfy_messages_published_failure",
+	})
+	MessagesCached = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "ntfy_messages_cached_total",
+	})
+	MessagePublishDurationMillis = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "ntfy_message_publish_duration_ms",
+	})
+	FirebasePublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "ntfy_firebase_published_success",
+	})
+	FirebasePublishedFailure = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "ntfy_firebase_published_failure",
+	})
+	EmailsPublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "ntfy_emails_sent_success",
+	})
+	EmailsPublishedFailure = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "ntfy_emails_sent_failure",
+	})
+	EmailsReceivedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "ntfy_emails_received_success",
+	})
+	EmailsReceivedFailure = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "ntfy_emails_received_failure",
+	})
+	CallsMadeSuccess = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "ntfy_calls_made_success",
+	})
+	CallsMadeFailure = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "ntfy_calls_made_failure",
+	})
+	UnifiedPushPublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "ntfy_unifiedpush_published_success",
+	})
+	MatrixPublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "ntfy_matrix_published_success",
+	})
+	MatrixPublishedFailure = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "ntfy_matrix_published_failure",
+	})
+	AttachmentsTotalSize = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "ntfy_attachments_total_size",
+	})
+	Visitors = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "ntfy_visitors_total",
+	})
+	Users = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "ntfy_users_total",
+	})
+	Subscribers = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "ntfy_subscribers_total",
+	})
+	Topics = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "ntfy_topics_total",
+	})
+	HTTPRequests = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "ntfy_http_requests_total",
+	}, []string{"http_code", "ntfy_code", "http_method"})
+)
+
+// init registers all collectors with the default Prometheus registry. Registration is
+// unconditional: the collectors are only ever exposed if the server mounts the /metrics handler,
+// so there is nothing to be gained by tying registration to the config.
+func init() {
+	prometheus.MustRegister(
+		MessagesPublishedSuccess,
+		MessagesPublishedFailure,
+		MessagesCached,
+		MessagePublishDurationMillis,
+		FirebasePublishedSuccess,
+		FirebasePublishedFailure,
+		EmailsPublishedSuccess,
+		EmailsPublishedFailure,
+		EmailsReceivedSuccess,
+		EmailsReceivedFailure,
+		CallsMadeSuccess,
+		CallsMadeFailure,
+		UnifiedPushPublishedSuccess,
+		MatrixPublishedSuccess,
+		MatrixPublishedFailure,
+		AttachmentsTotalSize,
+		Visitors,
+		Users,
+		Subscribers,
+		Topics,
+		HTTPRequests,
+	)
+}

metrics/metrics_test.go

@@ -0,0 +1,58 @@
+package metrics
+
+import (
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/require"
+)
+
+// expectedMetricNames is the exact set of metrics the server exposes. These names are a public
+// contract: renaming or dropping one silently breaks existing dashboards and alerts.
+var expectedMetricNames = []string{
+	"ntfy_attachments_total_size",
+	"ntfy_calls_made_failure",
+	"ntfy_calls_made_success",
+	"ntfy_emails_received_failure",
+	"ntfy_emails_received_success",
+	"ntfy_emails_sent_failure",
+	"ntfy_emails_sent_success",
+	"ntfy_firebase_published_failure",
+	"ntfy_firebase_published_success",
+	"ntfy_http_requests_total",
+	"ntfy_matrix_published_failure",
+	"ntfy_matrix_published_success",
+	"ntfy_message_publish_duration_ms",
+	"ntfy_messages_cached_total",
+	"ntfy_messages_published_failure",
+	"ntfy_messages_published_success",
+	"ntfy_subscribers_total",
+	"ntfy_topics_total",
+	"ntfy_unifiedpush_published_success",
+	"ntfy_users_total",
+	"ntfy_visitors_total",
+}
+
+func TestRegisteredMetricNames(t *testing.T) {
+	HTTPRequests.WithLabelValues("200", "20000", "GET").Inc()
+	families, err := prometheus.DefaultGatherer.Gather()
+	require.Nil(t, err)
+	names := make([]string, 0)
+	for _, family := range families {
+		if strings.HasPrefix(family.GetName(), "ntfy_") {
+			names = append(names, family.GetName())
+		}
+	}
+	sort.Strings(names)
+	require.Equal(t, expectedMetricNames, names)
+}
+
+func TestCollectors_NeverNil(t *testing.T) {
+	// Call sites update metrics unconditionally, even when metrics are disabled, so no collector
+	// may ever be nil
+	MessagesPublishedSuccess.Inc()
+	MessagesCached.Set(1)
+	HTTPRequests.WithLabelValues("200", "20000", "PUT").Inc()
+}

server/config.go

@@ -150,7 +150,6 @@ type Config struct {
 	TwilioVerifyBaseURL                  string
 	TwilioVerifyService                  string
 	TwilioCallFormat                     *template.Template
-	MetricsEnable                        bool
 	MetricsListenHTTP                    string
 	ProfileListenHTTP                    string
 	MessageDelayMin                      time.Duration

server/log.go

@@ -24,7 +24,6 @@ const (
 	tagSMTP      = "smtp"  // Receive email
 	tagEmail     = "email" // Send email
 	tagTwilio    = "twilio"
-	tagMessageCache = "message_cache"
 	tagStripe    = "stripe"
 	tagAccount   = "account"
 	tagManager   = "manager"

server/server.go

@@ -36,6 +36,7 @@ import (
 	"heckel.io/ntfy/v2/log"
 	"heckel.io/ntfy/v2/mail"
 	"heckel.io/ntfy/v2/message"
+	"heckel.io/ntfy/v2/metrics"
 	"heckel.io/ntfy/v2/model"
 	"heckel.io/ntfy/v2/payments"
 	"heckel.io/ntfy/v2/twilio"
@@ -411,13 +412,11 @@ func (s *Server) Run() error {
 		}()
 	}
 	if s.config.MetricsListenHTTP != "" {
-		initMetrics()
 		s.httpMetricsServer = &http.Server{Addr: s.config.MetricsListenHTTP, Handler: promhttp.Handler()}
 		go func() {
 			errChan <- s.httpMetricsServer.ListenAndServe()
 		}()
 	} else if s.config.EnableMetrics {
-		initMetrics()
 		s.metricsHandler = promhttp.Handler()
 	}
 	if s.config.ProfileListenHTTP != "" {
@@ -505,9 +504,7 @@ func (s *Server) handle(w http.ResponseWriter, r *http.Request) {
 				s.handleError(w, r, v, err)
 				return
 			}
-			if metricHTTPRequests != nil {
+			metrics.HTTPRequests.WithLabelValues("200", "20000", r.Method).Inc()
-				metricHTTPRequests.WithLabelValues("200", "20000", r.Method).Inc()
-			}
 		}).
 		Debug("HTTP request finished")
 }
@@ -517,9 +514,7 @@ func (s *Server) handleError(w http.ResponseWriter, r *http.Request, v *visitor,
 	if !ok {
 		httpErr = errHTTPInternalError
 	}
-	if metricHTTPRequests != nil {
+	metrics.HTTPRequests.WithLabelValues(strconv.Itoa(httpErr.HTTPCode), strconv.Itoa(httpErr.Code), r.Method).Inc()
-		metricHTTPRequests.WithLabelValues(fmt.Sprintf("%d", httpErr.HTTPCode), fmt.Sprintf("%d", httpErr.Code), r.Method).Inc()
-	}
 	isRateLimiting := util.Contains(rateLimitingErrorCodes, httpErr.HTTPCode)
 	isNormalError := strings.Contains(err.Error(), "i/o timeout") || util.Contains(normalErrorCodes, httpErr.HTTPCode)
 	ev := logvr(v, r).Err(err)
@@ -940,27 +935,27 @@ func (s *Server) handlePublishInternal(r *http.Request, v *visitor) (*model.Mess
 	s.messages++
 	s.mu.Unlock()
 	if unifiedpush {
-		minc(metricUnifiedPushPublishedSuccess)
+		metrics.UnifiedPushPublishedSuccess.Inc()
 	}
-	mset(metricMessagePublishDurationMillis, time.Since(start).Milliseconds())
+	metrics.MessagePublishDurationMillis.Set(float64(time.Since(start).Milliseconds()))
 	return m, nil
 }
 
 func (s *Server) handlePublish(w http.ResponseWriter, r *http.Request, v *visitor) error {
 	m, err := s.handlePublishInternal(r, v)
 	if err != nil {
-		minc(metricMessagesPublishedFailure)
+		metrics.MessagesPublishedFailure.Inc()
 		return err
 	}
-	minc(metricMessagesPublishedSuccess)
+	metrics.MessagesPublishedSuccess.Inc()
 	return s.writeJSON(w, m.ForJSON())
 }
 
 func (s *Server) handlePublishMatrix(w http.ResponseWriter, r *http.Request, v *visitor) error {
 	_, err := s.handlePublishInternal(r, v)
 	if err != nil {
-		minc(metricMessagesPublishedFailure)
+		metrics.MessagesPublishedFailure.Inc()
-		minc(metricMatrixPublishedFailure)
+		metrics.MatrixPublishedFailure.Inc()
 		if e, ok := err.(*errHTTP); ok && e.HTTPCode == errHTTPInsufficientStorageUnifiedPush.HTTPCode {
 			topic, err := fromContext[*topic](r, contextTopic)
 			if err != nil {
@@ -976,8 +971,8 @@ func (s *Server) handlePublishMatrix(w http.ResponseWriter, r *http.Request, v *
 		}
 		return err
 	}
-	minc(metricMessagesPublishedSuccess)
+	metrics.MessagesPublishedSuccess.Inc()
-	minc(metricMatrixPublishedSuccess)
+	metrics.MatrixPublishedSuccess.Inc()
 	return writeMatrixSuccess(w)
 }
 
@@ -1049,7 +1044,7 @@ func (s *Server) handleActionMessage(w http.ResponseWriter, r *http.Request, v *
 func (s *Server) sendToFirebase(v *visitor, m *model.Message) {
 	logvm(v, m).Tag(tagFirebase).Debug("Publishing to Firebase")
 	if err := s.firebaseClient.Send(v, m); err != nil {
-		minc(metricFirebasePublishedFailure)
+		metrics.FirebasePublishedFailure.Inc()
 		if errors.Is(err, errFirebaseTemporarilyBanned) {
 			logvm(v, m).Tag(tagFirebase).Err(err).Debug("Unable to publish to Firebase: %v", err.Error())
 		} else {
@@ -1057,17 +1052,17 @@ func (s *Server) sendToFirebase(v *visitor, m *model.Message) {
 		}
 		return
 	}
-	minc(metricFirebasePublishedSuccess)
+	metrics.FirebasePublishedSuccess.Inc()
 }
 
 func (s *Server) sendEmail(v *visitor, m *model.Message, email string) {
 	logvm(v, m).Tag(tagEmail).Field("email", email).Info("Sending email to %s", email)
 	if err := s.mailer.SendNotification(email, m, v.ip.String()); err != nil {
 		logvm(v, m).Tag(tagEmail).Field("email", email).Err(err).Warn("Unable to send email to %s: %v", email, err.Error())
-		minc(metricEmailsPublishedFailure)
+		metrics.EmailsPublishedFailure.Inc()
 		return
 	}
-	minc(metricEmailsPublishedSuccess)
+	metrics.EmailsPublishedSuccess.Inc()
 }
 
 func (s *Server) forwardPollRequest(v *visitor, m *model.Message) {

server/server.yml

@@ -423,8 +423,9 @@
 # doing, and/or secure access to the endpoint in your reverse proxy.
 #
 # - enable-metrics enables the /metrics endpoint for the default ntfy server (i.e. HTTP, HTTPS and/or Unix socket)
-# - metrics-listen-http exposes the metrics endpoint via a dedicated [IP]:port. If set, this option implicitly
+# - metrics-listen-http moves the metrics endpoint to a dedicated [IP]:port, e.g. "10.0.1.1:9090" or ":9090".
-#   enables metrics as well, e.g. "10.0.1.1:9090" or ":9090"
+#   It implicitly enables metrics. If set, the metrics are served only on that dedicated port, and the default
+#   ntfy server does not serve /metrics, even if enable-metrics is also set.
 #
 # enable-metrics: false
 # metrics-listen-http:

server/server_manager.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"heckel.io/ntfy/v2/log"
+	"heckel.io/ntfy/v2/metrics"
 	"heckel.io/ntfy/v2/util"
 )
 
@@ -93,13 +94,13 @@ func (s *Server) execManager() {
 			"emails_sent_failure":     sentMailFailure,
 		}).
 		Info("Server stats")
-	mset(metricMessagesCached, messagesCached)
+	metrics.MessagesCached.Set(float64(messagesCached))
-	mset(metricVisitors, visitorsCount)
+	metrics.Visitors.Set(float64(visitorsCount))
-	mset(metricUsers, usersCount)
+	metrics.Users.Set(float64(usersCount))
-	mset(metricSubscribers, subscribers)
+	metrics.Subscribers.Set(float64(subscribers))
-	mset(metricTopics, topicsCount)
+	metrics.Topics.Set(float64(topicsCount))
 	if s.attachment != nil {
-		mset(metricAttachmentsTotalSize, s.attachment.Size())
+		metrics.AttachmentsTotalSize.Set(float64(s.attachment.Size()))
 	}
 }
 

server/server_metrics.go

@@ -1,132 +0,0 @@
-package server
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-)
-
-var (
-	metricMessagesPublishedSuccess     prometheus.Counter
-	metricMessagesPublishedFailure     prometheus.Counter
-	metricMessagesCached               prometheus.Gauge
-	metricMessagePublishDurationMillis prometheus.Gauge
-	metricFirebasePublishedSuccess     prometheus.Counter
-	metricFirebasePublishedFailure     prometheus.Counter
-	metricEmailsPublishedSuccess       prometheus.Counter
-	metricEmailsPublishedFailure       prometheus.Counter
-	metricEmailsReceivedSuccess        prometheus.Counter
-	metricEmailsReceivedFailure        prometheus.Counter
-	metricCallsMadeSuccess             prometheus.Counter
-	metricCallsMadeFailure             prometheus.Counter
-	metricUnifiedPushPublishedSuccess  prometheus.Counter
-	metricMatrixPublishedSuccess       prometheus.Counter
-	metricMatrixPublishedFailure       prometheus.Counter
-	metricAttachmentsTotalSize         prometheus.Gauge
-	metricVisitors                     prometheus.Gauge
-	metricSubscribers                  prometheus.Gauge
-	metricTopics                       prometheus.Gauge
-	metricUsers                        prometheus.Gauge
-	metricHTTPRequests                 *prometheus.CounterVec
-)
-
-func initMetrics() {
-	metricMessagesPublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "ntfy_messages_published_success",
-	})
-	metricMessagesPublishedFailure = prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "ntfy_messages_published_failure",
-	})
-	metricMessagesCached = prometheus.NewGauge(prometheus.GaugeOpts{
-		Name: "ntfy_messages_cached_total",
-	})
-	metricMessagePublishDurationMillis = prometheus.NewGauge(prometheus.GaugeOpts{
-		Name: "ntfy_message_publish_duration_ms",
-	})
-	metricFirebasePublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "ntfy_firebase_published_success",
-	})
-	metricFirebasePublishedFailure = prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "ntfy_firebase_published_failure",
-	})
-	metricEmailsPublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "ntfy_emails_sent_success",
-	})
-	metricEmailsPublishedFailure = prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "ntfy_emails_sent_failure",
-	})
-	metricEmailsReceivedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "ntfy_emails_received_success",
-	})
-	metricEmailsReceivedFailure = prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "ntfy_emails_received_failure",
-	})
-	metricCallsMadeSuccess = prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "ntfy_calls_made_success",
-	})
-	metricCallsMadeFailure = prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "ntfy_calls_made_failure",
-	})
-	metricUnifiedPushPublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "ntfy_unifiedpush_published_success",
-	})
-	metricMatrixPublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "ntfy_matrix_published_success",
-	})
-	metricMatrixPublishedFailure = prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "ntfy_matrix_published_failure",
-	})
-	metricAttachmentsTotalSize = prometheus.NewGauge(prometheus.GaugeOpts{
-		Name: "ntfy_attachments_total_size",
-	})
-	metricVisitors = prometheus.NewGauge(prometheus.GaugeOpts{
-		Name: "ntfy_visitors_total",
-	})
-	metricUsers = prometheus.NewGauge(prometheus.GaugeOpts{
-		Name: "ntfy_users_total",
-	})
-	metricSubscribers = prometheus.NewGauge(prometheus.GaugeOpts{
-		Name: "ntfy_subscribers_total",
-	})
-	metricTopics = prometheus.NewGauge(prometheus.GaugeOpts{
-		Name: "ntfy_topics_total",
-	})
-	metricHTTPRequests = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Name: "ntfy_http_requests_total",
-	}, []string{"http_code", "ntfy_code", "http_method"})
-	prometheus.MustRegister(
-		metricMessagesPublishedSuccess,
-		metricMessagesPublishedFailure,
-		metricMessagesCached,
-		metricMessagePublishDurationMillis,
-		metricFirebasePublishedSuccess,
-		metricFirebasePublishedFailure,
-		metricEmailsPublishedSuccess,
-		metricEmailsPublishedFailure,
-		metricEmailsReceivedSuccess,
-		metricEmailsReceivedFailure,
-		metricCallsMadeSuccess,
-		metricCallsMadeFailure,
-		metricUnifiedPushPublishedSuccess,
-		metricMatrixPublishedSuccess,
-		metricMatrixPublishedFailure,
-		metricAttachmentsTotalSize,
-		metricVisitors,
-		metricUsers,
-		metricSubscribers,
-		metricTopics,
-		metricHTTPRequests,
-	)
-}
-
-// minc increments a prometheus.Counter if it is non-nil
-func minc(counter prometheus.Counter) {
-	if counter != nil {
-		counter.Inc()
-	}
-}
-
-// mset sets a prometheus.Gauge if it is non-nil
-func mset[T int | int64 | float64](gauge prometheus.Gauge, value T) {
-	if gauge != nil {
-		gauge.Set(float64(value))
-	}
-}

server/server_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/crypto/bcrypt"
 	dbtest "heckel.io/ntfy/v2/db/test"
@@ -323,6 +324,40 @@ func TestServer_WebEnabled(t *testing.T) {
 		require.Equal(t, 200, rr.Code)
 	})
 }
+
+// TestServer_MetricsEnabled ensures that the /metrics endpoint serves the registered ntfy metrics
+// once the metrics handler is set (as Serve does when enable-metrics is configured).
+func TestServer_MetricsEnabled(t *testing.T) {
+	forEachBackend(t, func(t *testing.T, databaseURL string) {
+		s := newTestServer(t, newTestConfig(t, databaseURL))
+		s.metricsHandler = promhttp.Handler() // Serve sets this when enable-metrics is configured
+
+		// Count at least one request first: Prometheus only reports a CounterVec such as
+		// ntfy_http_requests_total once it has children
+		request(t, s, "GET", "/v1/health", "", nil)
+
+		rr := request(t, s, "GET", "/metrics", "", nil)
+		require.Equal(t, 200, rr.Code)
+		require.Contains(t, rr.Body.String(), "ntfy_messages_published_success")
+		require.Contains(t, rr.Body.String(), "ntfy_http_requests_total")
+	})
+}
+
+// TestServer_MetricsDisabled ensures that the ntfy metrics are not exposed when the metrics handler
+// is unset (the default). The collectors are always registered with the Prometheus registry, so a
+// nil metrics handler is the only thing keeping them off the wire.
+func TestServer_MetricsDisabled(t *testing.T) {
+	forEachBackend(t, func(t *testing.T, databaseURL string) {
+		conf := newTestConfig(t, databaseURL)
+		conf.WebRoot = "" // Disable the web app, so its catch-all does not mask the /metrics route
+		s := newTestServer(t, conf)
+
+		rr := request(t, s, "GET", "/metrics", "", nil)
+		require.Equal(t, 404, rr.Code)
+		require.NotContains(t, rr.Body.String(), "ntfy_messages_published_success")
+	})
+}
+
 func TestServer_PublishLargeMessage(t *testing.T) {
 	forEachBackend(t, func(t *testing.T, databaseURL string) {
 		c := newTestConfig(t, databaseURL)
@@ -1684,6 +1719,7 @@ func TestServer_PublishEmailVerify_BoolValueUsesPrimary(t *testing.T) {
 			"Authorization": util.BasicAuth("phil", "phil"),
 		})
 		require.Equal(t, 200, response.Code)
+		waitFor(t, func() bool { return mailer.LastTo() != "" }) // E-Mail publishing happens in a Go routine
 		require.Equal(t, "zzz@example.com", mailer.LastTo())
 	})
 }
@@ -1710,6 +1746,7 @@ func TestServer_PublishEmailVerify_BoolValueNoVerifyUsesPrimary(t *testing.T) {
 			"Authorization": util.BasicAuth("phil", "phil"),
 		})
 		require.Equal(t, 200, response.Code)
+		waitFor(t, func() bool { return mailer.LastTo() != "" }) // E-Mail publishing happens in a Go routine
 		require.Equal(t, "zzz@example.com", mailer.LastTo())
 	})
 }
@@ -1751,6 +1788,7 @@ func TestServer_PublishEmailVerify_BoolValueProvisionedUsesPrimary(t *testing.T)
 			"Authorization": util.BasicAuth("prov", "provpass"),
 		})
 		require.Equal(t, 200, response.Code)
+		waitFor(t, func() bool { return mailer.LastTo() != "" }) // E-Mail publishing happens in a Go routine
 		require.Equal(t, "zzz@example.com", mailer.LastTo())
 	})
 }

server/server_twilio.go

@@ -1,6 +1,7 @@
 package server
 
 import (
+	"heckel.io/ntfy/v2/metrics"
 	"heckel.io/ntfy/v2/model"
 	"heckel.io/ntfy/v2/twilio"
 	"heckel.io/ntfy/v2/user"
@@ -46,8 +47,8 @@ func (s *Server) callPhone(v *visitor, m *model.Message, to string) {
 	})
 	if err != nil {
 		logvm(v, m).Tag(tagTwilio).Field("twilio_to", to).Err(err).Warn("Unable to call phone %s: %v", to, err.Error())
-		minc(metricCallsMadeFailure)
+		metrics.CallsMadeFailure.Inc()
 		return
 	}
-	minc(metricCallsMadeSuccess)
+	metrics.CallsMadeSuccess.Inc()
 }

server/smtp_server.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/emersion/go-smtp"
 	"github.com/microcosm-cc/bluemonday"
+	"heckel.io/ntfy/v2/metrics"
 	"heckel.io/ntfy/v2/model"
 )
 
@@ -180,7 +181,7 @@ func (s *smtpSession) Data(r io.Reader) error {
 		s.backend.mu.Lock()
 		s.backend.success++
 		s.backend.mu.Unlock()
-		minc(metricEmailsReceivedSuccess)
+		metrics.EmailsReceivedSuccess.Inc()
 		return nil
 	})
 }
@@ -238,7 +239,7 @@ func (s *smtpSession) withFailCount(fn func() error) error {
 		// We do not want to spam the log with WARN messages.
 		logem(s.conn).Err(err).Debug("Incoming mail error")
 		s.backend.failure++
-		minc(metricEmailsReceivedFailure)
+		metrics.EmailsReceivedFailure.Inc()
 	}
 	return err
 }

twilio/client.go

@@ -58,7 +58,7 @@ func (c *Client) Call(to string, data *CallData) error {
 		// Twilio rejects calls with a 4xx, e.g. for an invalid phone number, or if the account
 		// is out of funds. Without this check, a rejected call would be counted as a success.
 		ev.Field("twilio_status", code).Field("twilio_response", response).Warn("Twilio call failed with status code %d", code)
-		return err
+		return fmt.Errorf("twilio call failed with status code %d", code)
 	}
 	ev.FieldIf("twilio_response", response, log.TraceLevel).Debug("Received successful Twilio response")
 	return nil
@@ -80,7 +80,7 @@ func (c *Client) Verify(phoneNumber, channel string) error {
 		// Without this check, a rejected verification would look like a success to the caller,
 		// and the user would be told to wait for an SMS that was never sent.
 		ev.Field("twilio_status", code).Field("twilio_response", response).Warn("Twilio phone verification request failed with status code %d", code)
-		return err
+		return fmt.Errorf("twilio phone verification request failed with status code %d", code)
 	}
 	ev.FieldIf("twilio_response", response, log.TraceLevel).Debug("Received Twilio phone verification response")
 	return nil