WIP: Clsuter support (cross node delivery, leader election)

server/config.go

@@ -10,6 +10,8 @@ import (
 	"text/template"
 	"time"
 
+	"heckel.io/ntfy/v2/cluster"
+
 	"heckel.io/ntfy/v2/ban"
 	"heckel.io/ntfy/v2/user"
 )
@@ -119,8 +121,13 @@ type Config struct {
 	ListenUnixMode                       fs.FileMode
 	KeyFile                              string
 	CertFile                             string
-	DatabaseURL                          string   // PostgreSQL connection string (e.g. "postgres://user:pass@host:5432/ntfy")
-	DatabaseReplicaURLs                  []string // PostgreSQL read replica connection strings
+	DatabaseURL                          string        // PostgreSQL connection string (e.g. "postgres://user:pass@host:5432/ntfy")
+	DatabaseReplicaURLs                  []string      // PostgreSQL read replica connection strings
+	ClusterNodeID                        string        // Stable per-node identifier used to skip a node's own fan-out; required in cluster mode
+	ClusterListen                        string        // ip:port the dedicated cluster fan-out listener binds to (private network, e.g. "10.0.0.5:2587")
+	ClusterAdvertiseURL                  string        // Base URL peers use to reach this node's fan-out listener (defaults to "http://<cluster-listen>")
+	ClusterSecret                        string        `hash:"-"` // Shared secret authenticating node-to-node fan-out requests
+	ClusterBatchLinger                   time.Duration // How long fan-out messages wait to form a batch per peer; 0 sends immediately
 	FirebaseKeyFile                      string
 	CacheFile                            string
 	CacheDuration                        time.Duration
@@ -236,6 +243,11 @@ func NewConfig() *Config {
 		KeyFile:                              "",
 		CertFile:                             "",
 		DatabaseURL:                          "",
+		ClusterNodeID:                        "",
+		ClusterListen:                        "",
+		ClusterAdvertiseURL:                  "",
+		ClusterSecret:                        "",
+		ClusterBatchLinger:                   cluster.DefaultBatchLinger,
 		FirebaseKeyFile:                      "",
 		CacheFile:                            "",
 		CacheDuration:                        DefaultCacheDuration,

server/config_test.go

@@ -25,6 +25,7 @@ func TestConfig_HashExcludesSecrets(t *testing.T) {
 	conf2.UpstreamAccessToken = "tk_upstream"
 	conf2.WebPushPrivateKey = "web-push-private-key"
 	conf2.SMTPSenderPass = "hunter2"
+	conf2.ClusterSecret = "cluster-secret"
 	conf2.AuthUsers = []*user.User{{Name: "phil", Hash: "$2a$10$somebcrypthash"}}
 	conf2.AuthTokens = map[string][]*user.Token{"phil": {{Value: "tk_secrettoken"}}}
 	assert.Equal(t, conf1.Hash(), conf2.Hash())

server/server.go

@@ -32,6 +32,7 @@ import (
 	"heckel.io/ntfy/v2/action"
 	"heckel.io/ntfy/v2/attachment"
 	"heckel.io/ntfy/v2/ban"
+	"heckel.io/ntfy/v2/cluster"
 	"heckel.io/ntfy/v2/db"
 	"heckel.io/ntfy/v2/db/pg"
 	"heckel.io/ntfy/v2/log"
@@ -72,6 +73,8 @@ type Server struct {
 	stripe            stripeAPI                           // Stripe API, can be replaced with a mock
 	priceCache        *util.LookupCache[map[string]int64] // Stripe price ID -> price as cents (USD implied!)
 	metricsHandler    http.Handler                        // Handles /metrics if enable-metrics set, and listen-metrics-http not set
+	cluster           cluster.Cluster                     // Fans messages out to peer cluster nodes (nop when not clustered)
+	httpClusterServer *http.Server                        // Dedicated private listener for node-to-node fan-out (cluster-listen)
 	closeChan         chan bool
 	mu                sync.RWMutex
 }
@@ -334,9 +337,90 @@ func New(conf *Config) (*Server, error) {
 		stripe:          stripe,
 	}
 	s.priceCache = util.NewLookupCache(s.fetchStripePrices, conf.StripePriceCacheDuration)
+	// Cross-node cluster; delivery of peer messages to local subscribers is injected
+	// as a callback, so the cluster package never depends on the server. Peers talk to each
+	// other only via the dedicated cluster listener, never via the public listeners.
+	advertiseURL := conf.ClusterAdvertiseURL
+	if advertiseURL == "" && conf.ClusterListen != "" {
+		advertiseURL = "http://" + conf.ClusterListen
+	}
+	s.cluster, err = cluster.New(&cluster.Config{
+		Enabled:         conf.ClusterListen != "", // Setting cluster-listen implicitly enables clustering
+		NodeID:          cluster.NodeID(conf.ClusterNodeID),
+		AdvertiseURL:    advertiseURL,
+		Secret:          conf.ClusterSecret,
+		BatchLinger:     conf.ClusterBatchLinger,
+		MaxMessageBytes: int64(conf.MessageSizeLimit)*4 + 1024, // Envelope overhead over the raw message
+	}, pool, s.deliverFromBus, s.liveTopics)
+	if err != nil {
+		return nil, err
+	}
+	// The cluster routes messages by subscription knowledge: peers learn this node's live topics
+	// via periodic state pushes (liveTopics) and immediate announcements on a topic's first
+	// subscriber (the hook below; also set in topicsFromIDs for topics created later)
+	for _, t := range s.topics {
+		t.onFirstSubscriber = s.topicAnnouncer(t.ID)
+	}
 	return s, nil
 }
 
+// clusterHandler returns the handler served on the dedicated cluster listener (cluster-listen).
+// It serves the internal peer API (owned and routed by the cluster itself, including auth) plus
+// a health endpoint; the public listeners never expose these paths, so internal traffic cannot
+// be reached from the outside even before any firewalling.
+func (s *Server) clusterHandler() http.Handler {
+	mux := http.NewServeMux()
+	// TODO(T8): reflect s.cluster.Healthy() here (and in handleHealth) instead of a static
+	// response, so health checks can pull a node that lost its registry heartbeat
+	mux.HandleFunc(apiHealthPath, func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		io.WriteString(w, `{"healthy":true}`+"\n")
+	})
+	mux.Handle("/", s.cluster)
+	return mux
+}
+
+// liveTopics returns the topics that currently have at least one subscriber, computed fresh on
+// every call. Deliberately NOT the whole topics map: it also holds subscriber-less topics
+// rebuilt from the message cache, which would gut the routing filter's selectivity.
+func (s *Server) liveTopics() []string {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	topics := make([]string, 0, len(s.topics))
+	for _, t := range s.topics {
+		if subscribers, _ := t.Stats(); subscribers > 0 {
+			topics = append(topics, t.ID)
+		}
+	}
+	return topics
+}
+
+// topicAnnouncer returns the first-subscriber hook for a topic: it tells peer nodes right away
+// that this node now wants messages for it (see Cluster.AnnounceTopics).
+func (s *Server) topicAnnouncer(id string) func() {
+	return func() {
+		s.cluster.AnnounceTopics([]string{id})
+	}
+}
+
+// deliverFromBus delivers a message received from a peer node (via the cluster) to this
+// node's local subscribers. It is the receive-side counterpart to Cluster.Relay: local
+// delivery and all global side effects (Firebase, email, web push, upstream) already ran on the
+// origin node, so this only publishes to the local topic, and never re-relays.
+func (s *Server) deliverFromBus(m *model.Message) {
+	s.mu.RLock()
+	t, ok := s.topics[m.Topic]
+	s.mu.RUnlock()
+	if !ok {
+		metrics.ClusterMessagesWasted.Inc() // Relayed here needlessly: this node had no use for the message
+		return
+	}
+	v := s.visitor(m.Sender, nil)
+	if err := t.Publish(v, m); err != nil {
+		logvm(v, m).Err(err).Warn("Cluster: unable to deliver fan-out message to local subscribers")
+	}
+}
+
 func createMessageCache(conf *Config, pool *db.DB) (*message.Cache, error) {
 	if conf.CacheDuration == 0 {
 		return message.NewNopStore()
@@ -379,6 +463,9 @@ func (s *Server) Run() error {
 	if s.config.ProfileListenHTTP != "" {
 		listenStr += fmt.Sprintf(" %s[http/profile]", s.config.ProfileListenHTTP)
 	}
+	if s.config.ClusterListen != "" {
+		listenStr += fmt.Sprintf(" %s[http/cluster]", s.config.ClusterListen)
+	}
 	log.Tag(tagStartup).Info("Listening on%s, ntfy %s, log level is %s", listenStr, s.config.BuildVersion, log.CurrentLevel().String())
 	if log.IsFile() {
 		fmt.Fprintf(os.Stderr, "Listening on%s, ntfy %s\n", listenStr, s.config.BuildVersion)
@@ -433,6 +520,12 @@ func (s *Server) Run() error {
 	} else if s.config.EnableMetrics {
 		s.metricsHandler = promhttp.Handler()
 	}
+	if s.config.ClusterListen != "" {
+		s.httpClusterServer = &http.Server{Addr: s.config.ClusterListen, Handler: s.clusterHandler()}
+		go func() {
+			errChan <- s.httpClusterServer.ListenAndServe()
+		}()
+	}
 	if s.config.ProfileListenHTTP != "" {
 		profileMux := http.NewServeMux()
 		profileMux.HandleFunc("/debug/pprof/", pprof.Index)
@@ -478,6 +571,12 @@ func (s *Server) Stop() {
 	if s.attachment != nil {
 		s.attachment.Close()
 	}
+	if s.httpClusterServer != nil {
+		s.httpClusterServer.Close()
+	}
+	if s.cluster != nil {
+		s.cluster.Close()
+	}
 	s.closeDatabases()
 	if s.ban != nil {
 		s.ban.Close()
@@ -730,6 +829,8 @@ func (s *Server) handleTopicAuth(w http.ResponseWriter, _ *http.Request, _ *visi
 }
 
 func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request, _ *visitor) error {
+	// TODO(T8): in cluster mode, reflect s.cluster.Healthy() so DNS/LB health checks stop
+	// routing NEW clients to a node whose peers no longer relay to it (see Cluster interface)
 	response := &apiHealthResponse{
 		Healthy: true,
 	}
@@ -833,6 +934,60 @@ func (s *Server) handleMatrixDiscovery(w http.ResponseWriter) error {
 	return writeMatrixDiscoveryResponse(w)
 }
 
+// dispatchOpts selects which delivery targets fire for a published message, beyond delivery to
+// local subscribers and the cross-node broadcast (which always happen).
+type dispatchOpts struct {
+	firebase bool   // Send to Firebase (if configured)
+	email    string // Send an email to this address (if a mailer is configured)
+	call     string // Call this phone number (if Twilio is configured)
+	upstream bool   // Forward a poll request to the upstream server (if configured)
+	webPush  bool   // Publish to web push endpoints (if configured)
+	async    bool   // Deliver to local subscribers in a goroutine, logging errors instead of returning them
+}
+
+// dispatch delivers m to local subscribers, relays it to peer cluster nodes, and fires the
+// requested side-effect targets. It is the single choke point through which every published
+// message must pass; t may be nil when the topic has no local subscribers (delayed sender).
+//
+// The side-effect targets (Firebase, email, calls, upstream, web push) are global, not per-node:
+// they fire only here on the origin node, and never again when a peer node receives the message
+// via the cluster (see deliverFromBus).
+func (s *Server) dispatch(v *visitor, t *topic, m *model.Message, opts dispatchOpts) error {
+	// Deliver to local subscribers
+	if t != nil {
+		if opts.async {
+			go func() {
+				if err := t.Publish(v, m); err != nil {
+					logvm(v, m).Err(err).Warn("Unable to publish message")
+				}
+			}()
+		} else if err := t.Publish(v, m); err != nil {
+			return err
+		}
+	}
+	// Relay to peer cluster nodes, whose subscribers do not show up in this node's topics map
+	if err := s.cluster.Relay(m); err != nil {
+		logvm(v, m).Err(err).Warn("Cluster: unable to relay message to peer nodes")
+	}
+	// Fire the requested side-effect targets
+	if s.firebaseClient != nil && opts.firebase {
+		go s.sendToFirebase(v, m)
+	}
+	if s.mailer != nil && opts.email != "" {
+		go s.sendEmail(v, m, opts.email)
+	}
+	if s.config.TwilioAccount != "" && opts.call != "" {
+		go s.callPhone(v, m, opts.call)
+	}
+	if s.config.UpstreamBaseURL != "" && opts.upstream {
+		go s.forwardPollRequest(v, m)
+	}
+	if s.config.WebPushPublicKey != "" && opts.webPush {
+		go s.publishToWebPushEndpoints(v, m)
+	}
+	return nil
+}
+
 func (s *Server) handlePublishInternal(r *http.Request, v *visitor) (*model.Message, error) {
 	start := time.Now()
 	t, err := fromContext[*topic](r, contextTopic)
@@ -911,24 +1066,16 @@ func (s *Server) handlePublishInternal(r *http.Request, v *visitor) (*model.Mess
 		ev.Debug("Received message")
 	}
 	if !delayed {
-		if err := t.Publish(v, m); err != nil {
+		err := s.dispatch(v, t, m, dispatchOpts{
+			firebase: firebase,
+			email:    email,
+			call:     call,
+			upstream: !unifiedpush, // UP messages are not sent to upstream
+			webPush:  true,
+		})
+		if err != nil {
 			return nil, err
 		}
-		if s.firebaseClient != nil && firebase {
-			go s.sendToFirebase(v, m)
-		}
-		if s.mailer != nil && email != "" {
-			go s.sendEmail(v, m, email)
-		}
-		if s.config.TwilioAccount != "" && call != "" {
-			go s.callPhone(v, m, call)
-		}
-		if s.config.UpstreamBaseURL != "" && !unifiedpush { // UP messages are not sent to upstream
-			go s.forwardPollRequest(v, m)
-		}
-		if s.config.WebPushPublicKey != "" {
-			go s.publishToWebPushEndpoints(v, m)
-		}
 	} else {
 		logvrm(v, r, m).Tag(tagPublish).Debug("Message delayed, will process later")
 	}
@@ -1027,18 +1174,10 @@ func (s *Server) handleActionMessage(w http.ResponseWriter, r *http.Request, v *
 	m.Sender = v.IP()
 	m.User = v.MaybeUserID()
 	m.Expires = time.Unix(m.Time, 0).Add(v.Limits().MessageExpiryDuration).Unix()
-	// Publish to subscribers
-	if err := t.Publish(v, m); err != nil {
+	// Publish to subscribers, peer nodes, Firebase (for Android clients), and web push endpoints
+	if err := s.dispatch(v, t, m, dispatchOpts{firebase: true, webPush: true}); err != nil {
 		return err
 	}
-	// Send to Firebase for Android clients
-	if s.firebaseClient != nil {
-		go s.sendToFirebase(v, m)
-	}
-	// Send to web push endpoints
-	if s.config.WebPushPublicKey != "" {
-		go s.publishToWebPushEndpoints(v, m)
-	}
 	if event == model.MessageDeleteEvent {
 		// Delete any existing scheduled message with the same sequence ID
 		deletedIDs, err := s.messageCache.DeleteScheduledBySequenceID(t.ID, sequenceID)
@@ -1832,7 +1971,9 @@ func (s *Server) topicsFromIDs(v *visitor, ids ...string) ([]*topic, error) {
 			if v != nil && !v.TopicCreationAllowed() {
 				return nil, errHTTPTooManyRequestsLimitTopicCreation
 			}
-			s.topics[id] = newTopic(id)
+			t := newTopic(id)
+			t.onFirstSubscriber = s.topicAnnouncer(id)
+			s.topics[id] = t
 		}
 		topics = append(topics, s.topics[id])
 	}
@@ -1919,7 +2060,8 @@ func (s *Server) resetStats() {
 	for _, v := range s.visitors {
 		v.ResetStats()
 	}
-	if s.userManager != nil {
+	// The user database is shared; only the cluster leader resets it (always true single-node)
+	if s.userManager != nil && s.cluster.IsLeader() {
 		if err := s.userManager.ResetStats(); err != nil {
 			log.Tag(tagResetter).Warn("Failed to write to database: %s", err.Error())
 		}
@@ -1934,7 +2076,11 @@ func (s *Server) runFirebaseKeepaliver() {
 	for {
 		select {
 		case <-time.After(s.config.FirebaseKeepaliveInterval):
-			s.sendToFirebase(v, model.NewKeepaliveMessage(firebaseControlTopic))
+			// Leader only: every FCM keepalive wakes all subscribed phones, so a cluster must
+			// send it exactly once, not once per node (checked per tick to survive failover)
+			if s.cluster.IsLeader() {
+				s.sendToFirebase(v, model.NewKeepaliveMessage(firebaseControlTopic))
+			}
 		/*
 			FIXME: Disable iOS polling entirely for now due to thundering herd problem (see #677)
 			       To solve this, we'd have to shard the iOS poll topics to spread out the polling evenly.
@@ -1987,24 +2133,18 @@ func (s *Server) sendDelayedMessages() error {
 func (s *Server) sendDelayedMessage(v *visitor, m *model.Message) error {
 	logvm(v, m).Debug("Sending delayed message")
 	s.mu.RLock()
-	t, ok := s.topics[m.Topic] // If no subscribers, just mark message as published
+	t := s.topics[m.Topic] // May be nil if there are no local subscribers; dispatch handles that
 	s.mu.RUnlock()
-	if ok {
-		go func() {
-			// We do not rate-limit messages here, since we've rate limited them in the PUT/POST handler
-			if err := t.Publish(v, m); err != nil {
-				logvm(v, m).Err(err).Warn("Unable to publish message")
-			}
-		}()
-	}
-	if s.firebaseClient != nil { // Firebase subscribers may not show up in topics map
-		go s.sendToFirebase(v, m)
-	}
-	if s.config.UpstreamBaseURL != "" {
-		go s.forwardPollRequest(v, m)
-	}
-	if s.config.WebPushPublicKey != "" {
-		go s.publishToWebPushEndpoints(v, m)
+	// We do not rate-limit messages here, since we've rate limited them in the PUT/POST handler.
+	// Firebase subscribers may not show up in the topics map, so side effects fire regardless.
+	err := s.dispatch(v, t, m, dispatchOpts{
+		firebase: true,
+		upstream: true,
+		webPush:  true,
+		async:    true,
+	})
+	if err != nil {
+		return err
 	}
 	if err := s.messageCache.MarkPublished(m); err != nil {
 		return err

server/server.yml

@@ -61,6 +61,34 @@
 #
 # database-url: <connection-string>
 
+# If "cluster-listen" is set, clustering is implicitly enabled: this node registers itself in
+# the PostgreSQL node registry and fans published messages out to the other cluster nodes over
+# HTTP, so subscribers connected to any node receive messages published to any other node.
+# Requires "database-url" and "cluster-secret".
+#
+# - cluster-listen is the ip:port of the dedicated fan-out listener that peer nodes talk to.
+#   Bind it to a private network interface (e.g. "10.0.0.5:2587"); the public listeners never
+#   serve the fan-out endpoint.
+# - cluster-node-id is a stable per-node identifier (e.g. the hostname); required.
+# - cluster-advertise-url is the base URL peer nodes use to reach this node's fan-out listener;
+#   it defaults to "http://<cluster-listen>". It must be set explicitly if cluster-listen binds
+#   a wildcard address (e.g. ":2587").
+# - cluster-secret authenticates node-to-node fan-out requests; it must be identical on all
+#   nodes.
+# - cluster-batch-linger is how long fan-out messages may wait to form a batch per peer node,
+#   trading up to that much cross-node delivery latency for a bounded request rate between
+#   nodes. Set to 0 to send each message immediately.
+#
+# SECURITY: The fan-out endpoint injects messages into arbitrary topics. The shared secret
+# protects it, and it is only served on the dedicated cluster listener -- but you should still
+# make sure that listener is reachable only from the private network (firewall/VPC rules).
+#
+# cluster-listen: <ip:port>
+# cluster-node-id: <hostname>
+# cluster-advertise-url: "http://<cluster-listen>"
+# cluster-secret: <secret>
+# cluster-batch-linger: 500ms
+
 # If "cache-file" is set, messages are cached in a local SQLite database instead of only in-memory.
 # This allows for service restarts without losing messages in support of the since= parameter.
 # Not required if "database-url" is set (messages are stored in PostgreSQL instead).

server/server_account.go

@@ -985,7 +985,8 @@ func (s *Server) publishSyncEventForUser(v *visitor, u *user.User) error {
 		return err
 	}
 	m := model.NewDefaultMessage(syncTopic.ID, string(messageBytes))
-	if err := syncTopic.Publish(v, m); err != nil {
+	// Dispatch so the sync event also reaches the user's devices connected to peer cluster nodes
+	if err := s.dispatch(v, syncTopic, m, dispatchOpts{}); err != nil {
 		return err
 	}
 	return nil

server/server_cluster_test.go

@@ -0,0 +1,318 @@
+package server
+
+import (
+	"database/sql"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	dbtest "heckel.io/ntfy/v2/db/test"
+	"heckel.io/ntfy/v2/model"
+	"heckel.io/ntfy/v2/user"
+)
+
+// fakeCluster records relayed messages and topic announcements so tests can assert that every
+// publish path passes through the cluster exactly once, and that subscription hooks fire.
+type fakeCluster struct {
+	mu        sync.Mutex
+	messages  []*model.Message
+	announced []string
+	notLeader bool
+}
+
+func (b *fakeCluster) Relay(m *model.Message) error {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.messages = append(b.messages, m)
+	return nil
+}
+
+func (b *fakeCluster) ServeHTTP(_ http.ResponseWriter, _ *http.Request) {}
+
+func (b *fakeCluster) AnnounceTopics(topics []string) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.announced = append(b.announced, topics...)
+}
+
+func (b *fakeCluster) IsLeader() bool {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return !b.notLeader
+}
+
+func (b *fakeCluster) setLeader(leader bool) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.notLeader = !leader
+}
+
+func (b *fakeCluster) Close() error { return nil }
+
+func (b *fakeCluster) Messages() []*model.Message {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return append([]*model.Message{}, b.messages...)
+}
+
+func (b *fakeCluster) Announced() []string {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return append([]string{}, b.announced...)
+}
+
+func TestServer_Cluster_PublishRelaysOnce(t *testing.T) {
+	s := newTestServer(t, newTestConfig(t, ""))
+	b := &fakeCluster{}
+	s.cluster = b
+	response := request(t, s, "PUT", "/mytopic", "hi there", nil)
+	require.Equal(t, 200, response.Code)
+	messages := b.Messages()
+	require.Len(t, messages, 1)
+	require.Equal(t, "mytopic", messages[0].Topic)
+	require.Equal(t, "hi there", messages[0].Message)
+}
+
+func TestServer_Cluster_SyncEventRelays(t *testing.T) {
+	// Account sync events are delivered via the user's st_... sync topic; without relaying
+	// them, cross-device account sync silently breaks when a user's devices land on different
+	// cluster nodes.
+	s := newTestServer(t, newTestConfig(t, ""))
+	b := &fakeCluster{}
+	s.cluster = b
+	u := &user.User{ID: "u_abc", Name: "phil", SyncTopic: "st_1234"}
+	v := s.visitor(netip.MustParseAddr("1.2.3.4"), nil)
+	require.Nil(t, s.publishSyncEventForUser(v, u))
+	messages := b.Messages()
+	require.Len(t, messages, 1)
+	require.Equal(t, "st_1234", messages[0].Topic)
+}
+
+func TestServer_Cluster_DeliverNotOnPublicHandler(t *testing.T) {
+	// The fan-out endpoint lives only on the dedicated cluster listener; the public handler must
+	// not serve it, even with cluster mode on and a valid secret.
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	conf := newTestConfig(t, schemaDSN)
+	conf.ClusterNodeID = "node-a"
+	conf.ClusterListen = "127.0.0.1:1" // Enables clustering; not bound since Run() is not called
+	conf.ClusterSecret = "s3cret"
+	conf.ClusterAdvertiseURL = "http://127.0.0.1:1"
+	s := newTestServer(t, conf)
+	topics, err := s.topicsFromIDs(nil, "mytopic")
+	require.Nil(t, err)
+	var mu sync.Mutex
+	var received []*model.Message
+	topics[0].Subscribe(func(_ *visitor, m *model.Message) error {
+		mu.Lock()
+		defer mu.Unlock()
+		received = append(received, m)
+		return nil
+	}, "", func() {})
+	// A valid fan-out request against the PUBLIC handler must not deliver
+	response := request(t, s, "POST", "/v1/internal/message",
+		`{"message":{"id":"x1","time":1,"event":"message","topic":"mytopic","message":"sneaky"}}`,
+		map[string]string{"X-Cluster-Secret": "s3cret", "X-Cluster-Origin": "node-b"})
+	require.Equal(t, 404, response.Code)
+	time.Sleep(250 * time.Millisecond) // Delivery is async; give a wrong implementation time to fail
+	mu.Lock()
+	require.Empty(t, received)
+	mu.Unlock()
+	// The same request against the cluster listener handler DOES deliver
+	rr := httptest.NewRecorder()
+	req, err := http.NewRequest("POST", "/v1/internal/message",
+		strings.NewReader(`{"message":{"id":"x2","time":1,"event":"message","topic":"mytopic","message":"legit"}}`))
+	require.Nil(t, err)
+	req.Header.Set("X-Cluster-Secret", "s3cret")
+	req.Header.Set("X-Cluster-Origin", "node-b")
+	s.clusterHandler().ServeHTTP(rr, req)
+	require.Equal(t, 200, rr.Code)
+	waitFor(t, func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		return len(received) == 1
+	})
+}
+
+func TestServer_Cluster_EndToEnd(t *testing.T) {
+	// Two full servers sharing one Postgres schema: a message published to node A over HTTP must
+	// reach a subscriber connected to node B, via the node registry and the fan-out endpoint.
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	// Node B: create the listener first so its advertise URL is known before the server exists
+	listenerB, err := net.Listen("tcp", "127.0.0.1:0")
+	require.Nil(t, err)
+	confB := newTestConfig(t, schemaDSN)
+	confB.ClusterNodeID = "node-b"
+	confB.ClusterListen = listenerB.Addr().String() // Enables clustering; the test serves it below
+	confB.ClusterSecret = "s3cret"
+	confB.ClusterAdvertiseURL = "http://" + listenerB.Addr().String()
+	sB := newTestServer(t, confB)
+	srvB := &http.Server{Handler: sB.clusterHandler()}
+	go srvB.Serve(listenerB)
+	defer srvB.Close()
+	// Node A: publish-only in this test, so its advertise URL is never called
+	confA := newTestConfig(t, schemaDSN)
+	confA.ClusterNodeID = "node-a"
+	confA.ClusterListen = "127.0.0.1:1" // Enables clustering; not bound since Run() is not called
+	confA.ClusterSecret = "s3cret"
+	confA.ClusterAdvertiseURL = "http://127.0.0.1:1"
+	sA := newTestServer(t, confA)
+	// Subscribe on node B
+	topics, err := sB.topicsFromIDs(nil, "mytopic")
+	require.Nil(t, err)
+	var mu sync.Mutex
+	var received []*model.Message
+	topics[0].Subscribe(func(_ *visitor, m *model.Message) error {
+		mu.Lock()
+		defer mu.Unlock()
+		received = append(received, m)
+		return nil
+	}, "", func() {})
+	// Publish on node A
+	response := request(t, sA, "PUT", "/mytopic", "hello cluster", nil)
+	require.Equal(t, 200, response.Code)
+	waitFor(t, func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		return len(received) == 1
+	})
+	mu.Lock()
+	defer mu.Unlock()
+	require.Equal(t, "hello cluster", received[0].Message)
+}
+
+func TestServer_Cluster_DeliverFromBus(t *testing.T) {
+	// deliverFromBus is the receive side of the broadcaster: a message that originated on a peer
+	// node must reach this node's local subscribers, but must NOT be re-broadcast (loop) nor
+	// re-trigger origin-only side effects.
+	s := newTestServer(t, newTestConfig(t, ""))
+	b := &fakeCluster{}
+	s.cluster = b
+	topics, err := s.topicsFromIDs(nil, "mytopic")
+	require.Nil(t, err)
+	var mu sync.Mutex
+	var received []*model.Message
+	topics[0].Subscribe(func(_ *visitor, m *model.Message) error {
+		mu.Lock()
+		defer mu.Unlock()
+		received = append(received, m)
+		return nil
+	}, "", func() {})
+	m := model.NewDefaultMessage("mytopic", "from peer")
+	m.Sender = netip.MustParseAddr("5.6.7.8")
+	s.deliverFromBus(m)
+	waitFor(t, func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		return len(received) == 1
+	})
+	require.Empty(t, b.Messages()) // Peer messages are never re-relayed
+}
+
+func TestServer_Cluster_FirstSubscriberAnnounces(t *testing.T) {
+	// A topic gaining its FIRST subscriber is announced to peers exactly once, so publishers on
+	// other nodes stop skipping this node for it without waiting for the next state push.
+	s := newTestServer(t, newTestConfig(t, ""))
+	b := &fakeCluster{}
+	s.cluster = b
+	topics, err := s.topicsFromIDs(nil, "mytopic")
+	require.Nil(t, err)
+	subscriber := func(_ *visitor, _ *model.Message) error { return nil }
+	topics[0].Subscribe(subscriber, "", func() {})
+	waitFor(t, func() bool {
+		return len(b.Announced()) == 1 && b.Announced()[0] == "mytopic"
+	})
+	// A second subscriber does not re-announce
+	topics[0].Subscribe(subscriber, "", func() {})
+	time.Sleep(250 * time.Millisecond)
+	require.Len(t, b.Announced(), 1)
+}
+
+func TestServer_Cluster_ManagerPrunesOnlyOnLeader(t *testing.T) {
+	c := newTestConfig(t, "")
+	s := newTestServer(t, c)
+	cl := &fakeCluster{notLeader: true}
+	s.cluster = cl
+
+	// Publish and expire a message
+	rr := request(t, s, "POST", "/mytopic", "hi", nil)
+	require.Equal(t, 200, rr.Code)
+	m := toMessage(t, rr.Body.String())
+	require.Nil(t, s.messageCache.ExpireMessages("mytopic"))
+
+	// A non-leader node leaves shared-database pruning to the leader
+	s.execManager()
+	_, err := s.messageCache.Message(m.ID)
+	require.Nil(t, err)
+
+	// Once this node is the leader, the same run prunes
+	cl.setLeader(true)
+	s.execManager()
+	_, err = s.messageCache.Message(m.ID)
+	require.Equal(t, model.ErrMessageNotFound, err)
+}
+
+func TestServer_Cluster_StatsResetOnlyOnLeader(t *testing.T) {
+	c := newTestConfigWithAuthFile(t, "")
+	s := newTestServer(t, c)
+	cl := &fakeCluster{notLeader: true}
+	s.cluster = cl
+
+	// An anonymous visitor with an in-memory message count
+	v := newVisitor(c, s.messageCache, s.userManager, netip.MustParseAddr("1.2.3.4"), nil)
+	require.True(t, v.MessageAllowed())
+	s.mu.Lock()
+	s.visitors["ip:1.2.3.4"] = v
+	s.mu.Unlock()
+	require.Equal(t, int64(1), v.Stats().Messages)
+
+	// A user with persisted stats in the (shared) user database
+	require.Nil(t, s.userManager.AddUser("phil", "phil1234", user.RoleUser, false))
+	authDB, err := sql.Open("sqlite3", c.AuthFile)
+	require.Nil(t, err)
+	defer authDB.Close()
+	_, err = authDB.Exec(`UPDATE user SET stats_messages = 5 WHERE user = 'phil'`)
+	require.Nil(t, err)
+
+	// A non-leader node resets its own in-memory visitor stats, but leaves the user database
+	// to the leader
+	s.resetStats()
+	require.Equal(t, int64(0), v.Stats().Messages)
+	u, err := s.userManager.User("phil")
+	require.Nil(t, err)
+	require.Equal(t, int64(5), u.Stats.Messages)
+
+	// The leader resets the user database too
+	cl.setLeader(true)
+	s.resetStats()
+	u, err = s.userManager.User("phil")
+	require.Nil(t, err)
+	require.Equal(t, int64(0), u.Stats.Messages)
+}
+
+func TestServer_Cluster_FirebaseKeepaliverOnlyOnLeader(t *testing.T) {
+	// Every FCM keepalive wakes all subscribed phones, so only the leader may send them;
+	// N nodes sending N keepalives would multiply the battery cost for every user
+	c := newTestConfig(t, "")
+	c.FirebaseKeepaliveInterval = 20 * time.Millisecond
+	s := newTestServer(t, c)
+	sender := newTestFirebaseSender(100)
+	s.firebaseClient = newFirebaseClient(sender, &testAuther{Allow: true})
+	cl := &fakeCluster{notLeader: true}
+	s.cluster = cl
+	s.closeChan = make(chan bool) // Closed by Stop() in the test cleanup
+	go s.runFirebaseKeepaliver()
+
+	// A non-leader node stays silent
+	time.Sleep(150 * time.Millisecond)
+	require.Empty(t, sender.Messages())
+
+	// The leader sends keepalives
+	cl.setLeader(true)
+	waitFor(t, func() bool { return len(sender.Messages()) > 0 })
+}

server/server_manager.go

@@ -10,12 +10,16 @@ func (s *Server) execManager() {
 	// WARNING: Make sure to only selectively lock with the mutex, and be aware that this
 	//          there is no mutex for the entire function.
 
-	// Prune all the things
+	// Prune all the things. In-memory state is pruned on every node; jobs touching shared
+	// databases (and the web push job, which also sends expiry-warning notifications) run on
+	// the cluster leader only. In a single-node setup, IsLeader is always true.
 	s.pruneVisitors()
-	s.pruneTokens()
-	s.pruneAttachments()
-	s.pruneMessages()
-	s.pruneAndNotifyWebPushSubscriptions()
+	if s.cluster.IsLeader() {
+		s.pruneTokens()
+		s.pruneAttachments()
+		s.pruneMessages()
+		s.pruneAndNotifyWebPushSubscriptions()
+	}
 
 	// Message count
 	messagesCached, err := s.messageCache.MessagesCount()

server/topic.go

@@ -20,11 +20,12 @@ const (
 // topic represents a channel to which subscribers can subscribe, and publishers
 // can publish a message
 type topic struct {
-	ID          string
-	subscribers map[int]*topicSubscriber
-	rateVisitor *visitor
-	lastAccess  time.Time
-	mu          sync.RWMutex
+	ID                string
+	subscribers       map[int]*topicSubscriber
+	rateVisitor       *visitor
+	lastAccess        time.Time
+	onFirstSubscriber func() // Fired (async) when the subscriber count goes 0 -> 1; may be nil
+	mu                sync.RWMutex
 }
 
 type topicSubscriber struct {
@@ -56,6 +57,10 @@ func (t *topic) Subscribe(s subscriber, userID string, cancel func()) (subscribe
 			break
 		}
 	}
+	if len(t.subscribers) == 0 && t.onFirstSubscriber != nil {
+		// Fired async so cluster announcements never run under the topic lock
+		go t.onFirstSubscriber()
+	}
 	t.subscribers[subscriberID] = &topicSubscriber{
 		userID:     userID, // May be empty
 		subscriber: s,