Refactor

cluster/cluster.go

@@ -61,52 +61,26 @@ const (
 	DefaultBatchLinger = 500 * time.Millisecond
 )
 
-// Config configures the cluster. It is assembled by the server from its own config, which keeps
-// this package free of server types.
-type Config struct {
-	Enabled           bool          // Master switch; when false, New returns the nop cluster
-	NodeID            NodeID        // Stable per-node identifier; required
-	AdvertiseURL      string        // Base URL peers use to reach this node's fan-out endpoint
-	Secret            string        // Shared secret authenticating node-to-node fan-out requests
-	HeartbeatInterval time.Duration // How often the node registry heartbeat is refreshed
-	NodeTTL           time.Duration // Registry rows older than this do not count as live peers
-	BatchLinger       time.Duration // How long messages wait in a peer queue to form a batch; 0 = send immediately
-	StateInterval     time.Duration // How often the full subscription state is pushed to peers
-	MaxMessageBytes   int64         // Upper bound for a single message on the wire (batch limits derive from this)
-}
-
-// DeliverFunc hands a message received from a peer node to this node's local subscribers. The
-// server supplies it, which inverts the dependency: this package never imports the server.
-type DeliverFunc func(m *model.Message)
-
-// TopicsFunc returns the topics that currently have at least one live subscriber, computed
-// fresh on every call: membership is never tracked as a list, so topics "leave" simply by not
-// appearing in the next snapshot. The server supplies it (same inversion as DeliverFunc).
-type TopicsFunc func() []string
-
 // Cluster fans published messages out to peer cluster nodes and receives their fan-out requests.
 // Local delivery to a node's own subscribers still happens inline in the server; the cluster
 // only covers the cross-node hop.
 type Cluster interface {
-	// The internal peer API (/v1/internal/*), served on the dedicated cluster listener. The
-	// nop cluster answers 404.
 	http.Handler
-	// Relay sends a locally published message on to the peer nodes that may have subscribers
+	// ForwardMessage sends a locally published message on to the peer nodes that may have subscribers
 	// for its topic (all of them, when subscription knowledge is missing or stale). It is
 	// fire-and-forget and must not block the caller's request path.
-	Relay(m *model.Message) error
-	// AnnounceTopics tells peers that these topics just gained their first local subscriber,
-	// closing the routing-knowledge window to ~one round trip. Nop in single-node mode.
-	AnnounceTopics(topics []string)
+	ForwardMessage(m *model.Message) error
+	// BroadcastState pushes a subscription-state delta to ALL peers (unlike ForwardMessage,
+	// which routes), closing the routing-knowledge window to ~one round trip. Nop single-node.
+	BroadcastState(state *State)
 	// IsLeader reports whether this node holds the cluster leader lock. Singleton background
 	// jobs (e.g. the Firebase keepaliver) are gated on the leader.
 	IsLeader() bool
-	// TODO(T8): add Healthy() bool reflecting registration health (unhealthy when the last
-	// successful Register is older than NodeTTL, i.e. when peers stop relaying to this node),
-	// so /v1/health can pull a DB-partitioned node out of DNS rotation. Needs a fail-open
-	// answer at the health checker first: during a full DB outage ALL nodes fail to register
-	// while the mesh keeps delivering on stale peer caches, and naively pulling every node
-	// would turn a control-plane blip into a total outage.
+	// Healthy reports whether this node is fit to serve: its registry heartbeat is fresh
+	// enough (within NodeTTL) that peers still forward messages to it. Health checkers must
+	// fail open (never pull ALL nodes): during a full database outage every node reports
+	// unhealthy while the mesh keeps delivering on stale peer caches.
+	Healthy() bool
 	// Close stops the cluster and releases its resources.
 	Close() error
 }

cluster/cluster_load_test.go

@@ -67,7 +67,7 @@ func TestMesh_Soak(t *testing.T) {
 			ticker := time.NewTicker(publishInterval)
 			defer ticker.Stop()
 			for i := 0; i < messagesPerPublisher; i++ {
-				require.Nil(t, mesh.Relay(model.NewDefaultMessage("mytopic", fmt.Sprintf("p%d-m%d", p, i))))
+				require.Nil(t, mesh.ForwardMessage(model.NewDefaultMessage("mytopic", fmt.Sprintf("p%d-m%d", p, i))))
 				<-ticker.C
 			}
 		}(p)
@@ -89,10 +89,10 @@ func TestMesh_Soak(t *testing.T) {
 		total, requests, float64(total)/float64(requests), float64(total)/elapsed.Seconds())
 }
 
-// BenchmarkRelay measures the publish-path cost of Relay: marshal + peer lookup (cached)
+// BenchmarkForwardMessage measures the publish-path cost of ForwardMessage: marshal + peer lookup (cached)
 // + enqueue. The peer never drains, so enqueued fragments are dropped once the queue fills;
 // the benchmark measures the hot path, not HTTP delivery.
-func BenchmarkRelay(b *testing.B) {
+func BenchmarkForwardMessage(b *testing.B) {
 	if os.Getenv("NTFY_TEST_DATABASE_URL") == "" {
 		b.Skip("NTFY_TEST_DATABASE_URL not set")
 	}
@@ -107,7 +107,7 @@ func BenchmarkRelay(b *testing.B) {
 	m := model.NewDefaultMessage("mytopic", "benchmark message body of typical size for a push")
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		if err := mesh.Relay(m); err != nil {
+		if err := mesh.ForwardMessage(m); err != nil {
 			b.Fatal(err)
 		}
 	}

cluster/cluster_mesh.go

@@ -37,22 +37,24 @@ const (
 // Each peer has its own bounded send queue and delivery worker, so a slow or wedged peer only
 // backs up (and eventually drops) its own queue and never delays delivery to healthy peers.
 type meshCluster struct {
-	conf          *Config
-	deliver       DeliverFunc
-	topics        TopicsFunc
-	registry      *registry.Registry
-	leader        *pg.Leader
-	httpClient    *http.Client
-	mux           *http.ServeMux        // The internal peer API; Cluster is an http.Handler
-	queues        map[NodeID]*peerQueue // per-peer send queues; reconciled against the registry
-	closed        bool                  // Guards against Relay spawning new workers after Close
-	states        map[NodeID]*peerState // what each peer last told us (subscription knowledge)
-	lastStatePush time.Time             // Only touched by the heartbeat goroutine
-	ctx           context.Context
-	cancel        context.CancelFunc
-	wg            sync.WaitGroup
-	mu            sync.Mutex // Protects queues and closed
-	statesMu      sync.Mutex // Protects states
+	conf           *Config
+	deliver        DeliverFunc
+	topics         TopicsFunc
+	registry       *registry.Registry
+	leader         *pg.Leader
+	httpClient     *http.Client
+	mux            *http.ServeMux        // The internal peer API; Cluster is an http.Handler
+	queues         map[NodeID]*peerQueue // per-peer send queues; reconciled against the registry
+	closed         bool                  // Guards against ForwardMessage spawning new workers after Close
+	states         map[NodeID]*peerState // what each peer last told us (subscription knowledge)
+	lastStatePush  time.Time             // Only touched by the heartbeat goroutine
+	knownPeers     map[NodeID]string     // Peers seen in the last reconcile, for join/leave logging
+	lastRegistered time.Time             // Last successful registry heartbeat, for Healthy
+	ctx            context.Context
+	cancel         context.CancelFunc
+	wg             sync.WaitGroup
+	mu             sync.Mutex // Protects queues, closed, knownPeers and lastRegistered
+	statesMu       sync.Mutex // Protects states
 }
 
 // newMeshCluster creates the mesh cluster: it sets up the registry schema, registers this node
@@ -73,16 +75,19 @@ func newMeshCluster(conf *Config, pool *db.DB, deliver DeliverFunc, topics Topic
 	}
 	ctx, cancel := context.WithCancel(context.Background())
 	c := &meshCluster{
-		conf:       conf,
-		deliver:    deliver,
-		topics:     topics,
-		registry:   reg,
-		leader:     pg.NewLeader(pool.Primary(), pg.LeaderLockKey),
-		httpClient: &http.Client{Timeout: meshHTTPTimeout},
-		queues:     make(map[NodeID]*peerQueue),
-		states:     make(map[NodeID]*peerState),
-		ctx:        ctx,
-		cancel:     cancel,
+		conf:     conf,
+		deliver:  deliver,
+		topics:   topics,
+		registry: reg,
+		// Renews its lease on its own fixed cadence; see pg.Leader for the semantics
+		leader:         pg.NewLeader(pool.Primary(), pg.LeaderLockKey, conf.LeaderRenewInterval),
+		httpClient:     &http.Client{Timeout: meshHTTPTimeout},
+		queues:         make(map[NodeID]*peerQueue),
+		lastRegistered: time.Now(), // The synchronous Register above just succeeded
+		states:         make(map[NodeID]*peerState),
+		knownPeers:     make(map[NodeID]string),
+		ctx:            ctx,
+		cancel:         cancel,
 	}
 	c.mux = http.NewServeMux()
 	c.mux.HandleFunc("POST "+MessagePath, c.authenticated(c.handleMessage))
@@ -147,16 +152,19 @@ func (c *meshCluster) heartbeatLoop() {
 // periodically push our subscription state to peers.
 //
 // A node that cannot even register itself aborts the tick: the remaining database work would
-// fail against the same database, and everything downstream degrades safely without it -- Relay
+// fail against the same database, and everything downstream degrades safely without it -- ForwardMessage
 // serves the stale peer cache on its own, and peers fall back to broadcasting to us once our
 // last pushed state expires.
 func (c *meshCluster) heartbeat() error {
-	// TODO(T8): record the last successful Register here to back the future Healthy() method
-	// (see the Cluster interface)
 	if err := c.registry.Register(); err != nil {
 		return err
 	}
-	if c.leader.TryAcquire(c.ctx) {
+	c.mu.Lock()
+	c.lastRegistered = time.Now()
+	c.mu.Unlock()
+	// Effective leadership: pg.Leader's lease semantics guarantee a no-leader gap on
+	// failover, never two leaders
+	if c.leader.IsLeader() {
 		metrics.ClusterLeader.Set(1)
 		if err := c.registry.Prune(); err != nil {
 			log.Tag(tag).Err(err).Warn("Failed to prune stale nodes") // Housekeeping only; not fatal for the tick
@@ -179,7 +187,7 @@ func (c *meshCluster) heartbeat() error {
 // reconcilePeers aligns this node's per-peer attachments with the live peer set: it retires the
 // queues (and workers) of peers that have left the registry or re-registered under a new
 // advertise URL (the retired queue's remainder was headed for a dead address anyway), and prunes
-// the stale state of departed peers. New and replacement queues are created lazily by Relay, not
+// the stale state of departed peers. New and replacement queues are created lazily by ForwardMessage, not
 // here, so a freshly joined peer is reachable immediately.
 func (c *meshCluster) reconcilePeers(peers []*registry.Peer) {
 	metrics.ClusterPeers.Set(float64(len(peers)))
@@ -188,6 +196,18 @@ func (c *meshCluster) reconcilePeers(peers []*registry.Peer) {
 		alive[NodeID(p.NodeID)] = p.AdvertiseURL
 	}
 	c.mu.Lock()
+	// Log joins and leaves (as seen through the up-to-NodeTTL-stale registry view)
+	for nodeID, url := range alive {
+		if _, ok := c.knownPeers[nodeID]; !ok {
+			log.Tag(tag).Info("Peer %s (%s) joined the cluster", nodeID, url)
+		}
+	}
+	for nodeID := range c.knownPeers {
+		if _, ok := alive[nodeID]; !ok {
+			log.Tag(tag).Info("Peer %s left the cluster", nodeID)
+		}
+	}
+	c.knownPeers = alive
 	for nodeID, q := range c.queues {
 		if url, ok := alive[nodeID]; !ok || q.advertiseURL != url {
 			q.queue.Close() // Flushes the remainder; the worker exits when the queue is drained
@@ -227,11 +247,11 @@ func (c *meshCluster) queueFor(p *registry.Peer) *peerQueue {
 	return q
 }
 
-// Relay enqueues the message for delivery to every live peer node that may have subscribers for
+// ForwardMessage enqueues the message for delivery to every live peer node that may have subscribers for
 // its topic (all of them, absent fresh knowledge). Delivery is fire-and-forget via each peer's
 // bounded batching queue; if a peer's queue is full the message is dropped for that peer
 // (subscribers reconnect and re-poll history from the database).
-func (c *meshCluster) Relay(msg *model.Message) error {
+func (c *meshCluster) ForwardMessage(msg *model.Message) error {
 	peers, err := c.registry.Peers()
 	if err != nil {
 		return err
@@ -243,7 +263,7 @@ func (c *meshCluster) Relay(msg *model.Message) error {
 	if err != nil {
 		return err
 	}
-	metrics.ClusterMessagesRelayed.Inc()
+	metrics.ClusterMessagesForwarded.Inc()
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	if c.closed {
@@ -254,11 +274,16 @@ func (c *meshCluster) Relay(msg *model.Message) error {
 		// certain (no state, stale state) falls back to broadcasting
 		if !c.mayNeed(NodeID(p.NodeID), msg.Topic) {
 			metrics.ClusterRouteSkipped.Inc()
+			if ev := log.Tag(tag); ev.IsTrace() {
+				ev.Trace("Skipping peer %s for message %s: no subscribers for topic %s", p.NodeID, msg.ID, msg.Topic)
+			}
 			continue
 		}
 		if !c.queueFor(p).queue.TryEnqueue(frag) {
 			metrics.ClusterQueueDropped.Inc()
 			log.Tag(tag).Warn("Fan-out queue for peer %s full, dropping message %s", p.NodeID, msg.ID)
+		} else if ev := log.Tag(tag); ev.IsTrace() {
+			ev.Trace("Enqueued message %s (topic %s) for peer %s", msg.ID, msg.Topic, p.NodeID)
 		}
 	}
 	return nil
@@ -284,7 +309,9 @@ func (c *meshCluster) mayNeed(peer NodeID, topic string) bool {
 func (c *meshCluster) peerWorker(nodeID NodeID, q *peerQueue) {
 	defer c.wg.Done()
 	for frags := range q.queue.Dequeue() {
-		c.postToPeer(nodeID, messageURL(q.advertiseURL), contentTypeNDJSON, assembleMessageBody(frags))
+		body := assembleMessageBody(frags)
+		log.Tag(tag).Debug("Sending batch of %d message(s) (%d bytes) to peer %s", len(frags), len(body), nodeID)
+		c.postToPeer(nodeID, messageURL(q.advertiseURL), contentTypeNDJSON, body)
 		metrics.ClusterBatchesSent.Inc()
 	}
 }
@@ -319,13 +346,22 @@ func (c *meshCluster) postToPeer(nodeID NodeID, url, contentType string, payload
 
 // handleMessage receives a batch of peer messages (NDJSON) and streams them to local
 // subscribers line by line, delivering each message as it is decoded.
-func (c *meshCluster) handleMessage(_ NodeID, w http.ResponseWriter, r *http.Request) {
+func (c *meshCluster) handleMessage(origin NodeID, w http.ResponseWriter, r *http.Request) {
 	// A batch can exceed its byte cap by one message, plus framing overhead
 	maxBodyBytes := int64(batchMaxBytes) + c.conf.MaxMessageBytes + 1024
-	if err := decodeMessageBody(io.LimitReader(r.Body, maxBodyBytes), int(c.conf.MaxMessageBytes), c.deliver); err != nil {
+	received := 0
+	deliver := func(m *model.Message) {
+		received++
+		if ev := log.Tag(tag); ev.IsTrace() {
+			ev.Trace("Delivering message %s (topic %s) from peer %s", m.ID, m.Topic, origin)
+		}
+		c.deliver(m)
+	}
+	if err := decodeMessageBody(io.LimitReader(r.Body, maxBodyBytes), int(c.conf.MaxMessageBytes), deliver); err != nil {
 		w.WriteHeader(http.StatusBadRequest)
 		return
 	}
+	log.Tag(tag).Debug("Received batch of %d message(s) from peer %s", received, origin)
 	w.WriteHeader(http.StatusOK)
 }
 
@@ -362,6 +398,7 @@ func (c *meshCluster) applyTopicState(origin NodeID, topics *apiStateTopics) err
 			return err
 		}
 		c.states[origin] = &peerState{topics: filter, updatedAt: time.Now()}
+		log.Tag(tag).Debug("Received subscription state from peer %s (%d filter bytes)", origin, len(topics.Filter))
 		return nil
 	}
 	if state, ok := c.states[origin]; ok {
@@ -369,6 +406,7 @@ func (c *meshCluster) applyTopicState(origin NodeID, topics *apiStateTopics) err
 			state.topics.Add(topic)
 		}
 		state.updatedAt = time.Now()
+		log.Tag(tag).Debug("Received %d announced topic(s) from peer %s", len(topics.Added), origin)
 	}
 	return nil
 }
@@ -394,27 +432,29 @@ func (c *meshCluster) pushState(peers []*registry.Peer) {
 	if err != nil {
 		return
 	}
+	log.Tag(tag).Debug("Pushing subscription state (%d topics, %d bytes) to %d peer(s)", len(topics), len(body), len(peers))
 	for _, p := range peers {
 		go c.postToPeer(NodeID(p.NodeID), stateURL(p.AdvertiseURL), contentTypeJSON, body)
 	}
 	metrics.ClusterStatePushes.Inc()
 }
 
-// AnnounceTopics immediately tells all live peers that these topics gained their first local
+// BroadcastState immediately tells all live peers that these topics gained their first local
 // subscriber, shrinking the window in which a publisher could wrongly skip this node from a
 // full state interval down to about one round trip.
-func (c *meshCluster) AnnounceTopics(topics []string) {
-	if len(topics) == 0 {
+func (c *meshCluster) BroadcastState(state *State) {
+	if len(state.AddedTopics) == 0 {
 		return
 	}
 	peers, err := c.registry.Peers()
 	if err != nil || len(peers) == 0 {
 		return
 	}
-	body, err := json.Marshal(&apiState{Topics: &apiStateTopics{Added: topics}})
+	body, err := json.Marshal(&apiState{Topics: &apiStateTopics{Added: state.AddedTopics}})
 	if err != nil {
 		return
 	}
+	log.Tag(tag).Debug("Broadcasting state (%d new topics) to %d peer(s)", len(state.AddedTopics), len(peers))
 	for _, p := range peers {
 		go c.postToPeer(NodeID(p.NodeID), stateURL(p.AdvertiseURL), contentTypeJSON, body)
 	}
@@ -425,6 +465,14 @@ func (c *meshCluster) IsLeader() bool {
 	return c.leader.IsLeader()
 }
 
+// Healthy reports whether this node's registry heartbeat is fresh enough that peers still
+// forward messages to it (see the Cluster interface for the checker's fail-open duty).
+func (c *meshCluster) Healthy() bool {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return time.Since(c.lastRegistered) < c.conf.NodeTTL
+}
+
 // Close stops the mesh: it deregisters this node, releases leadership, stops all peer workers,
 // and waits for them to exit.
 func (c *meshCluster) Close() error {
@@ -444,7 +492,7 @@ func (c *meshCluster) Close() error {
 	if err := c.registry.Deregister(); err != nil {
 		log.Tag(tag).Err(err).Warn("Failed to deregister node")
 	}
-	c.leader.Release()
+	c.leader.Close()
 	metrics.ClusterLeader.Set(0)
 	return nil
 }

cluster/cluster_mesh_test.go

@@ -37,14 +37,15 @@ func openTestPool(t testing.TB, dsn string) *db.DB {
 
 func newTestMeshConfig(nodeID, advertiseURL string) *Config {
 	return &Config{
-		Enabled:           true,
-		NodeID:            NodeID(nodeID),
-		AdvertiseURL:      advertiseURL,
-		Secret:            testSecret,
-		HeartbeatInterval: 100 * time.Millisecond,
-		NodeTTL:           time.Second, // Also the peer cache bound; short so fake peers registered mid-test are seen quickly
-		MaxMessageBytes:   1 << 20,
-		StateInterval:     time.Minute, // Individual tests lower this to exercise state pushes
+		Enabled:             true,
+		NodeID:              NodeID(nodeID),
+		AdvertiseURL:        advertiseURL,
+		Secret:              testSecret,
+		HeartbeatInterval:   100 * time.Millisecond,
+		LeaderRenewInterval: 20 * time.Millisecond, // Lease duration 60ms, hold-off 120ms; keeps leadership tests fast
+		NodeTTL:             time.Second,           // Also the peer cache bound; short so fake peers registered mid-test are seen quickly
+		MaxMessageBytes:     1 << 20,
+		StateInterval:       time.Minute, // Individual tests lower this to exercise state pushes
 	}
 }
 
@@ -92,7 +93,7 @@ func TestMesh_CrossNodeDelivery(t *testing.T) {
 	require.Nil(t, err)
 	defer meshA.Close()
 	msg := model.NewDefaultMessage("mytopic", "hello cross-node")
-	require.Nil(t, meshA.Relay(msg))
+	require.Nil(t, meshA.ForwardMessage(msg))
 	waitFor(t, func() bool {
 		mu.Lock()
 		defer mu.Unlock()
@@ -207,7 +208,7 @@ func TestMesh_SlowPeerIsolation(t *testing.T) {
 	defer mesh.Close()
 	const n = 20
 	for i := 0; i < n; i++ {
-		require.Nil(t, mesh.Relay(model.NewDefaultMessage("mytopic", fmt.Sprintf("message %d", i))))
+		require.Nil(t, mesh.ForwardMessage(model.NewDefaultMessage("mytopic", fmt.Sprintf("message %d", i))))
 	}
 	waitFor(t, func() bool {
 		mu.Lock()
@@ -243,7 +244,7 @@ func TestMesh_BatchCoalescing(t *testing.T) {
 	defer mesh.Close()
 	const n = 20
 	for i := 0; i < n; i++ {
-		require.Nil(t, mesh.Relay(model.NewDefaultMessage("mytopic", fmt.Sprintf("message %d", i))))
+		require.Nil(t, mesh.ForwardMessage(model.NewDefaultMessage("mytopic", fmt.Sprintf("message %d", i))))
 	}
 	waitFor(t, func() bool {
 		mu.Lock()
@@ -281,7 +282,7 @@ func TestMesh_DeadPeerRemovedAndRejoin(t *testing.T) {
 	defer mesh.Close()
 	// The fake peer registers once and then "dies": its heartbeat is never refreshed
 	registerFakePeer(t, pool, "node-dead", srv.URL)
-	require.Nil(t, mesh.Relay(model.NewDefaultMessage("mytopic", "while alive")))
+	require.Nil(t, mesh.ForwardMessage(model.NewDefaultMessage("mytopic", "while alive")))
 	waitFor(t, func() bool {
 		mu.Lock()
 		defer mu.Unlock()
@@ -299,7 +300,7 @@ func TestMesh_DeadPeerRemovedAndRejoin(t *testing.T) {
 		require.Nil(t, pool.QueryRow(`SELECT COUNT(*) FROM node_registry WHERE node_id = 'node-dead'`).Scan(&count))
 		return count == 0
 	})
-	require.Nil(t, mesh.Relay(model.NewDefaultMessage("mytopic", "while dead")))
+	require.Nil(t, mesh.ForwardMessage(model.NewDefaultMessage("mytopic", "while dead")))
 	time.Sleep(250 * time.Millisecond) // Give a wrong implementation time to deliver anyway
 	mu.Lock()
 	require.Equal(t, 1, received) // Only the first message arrived
@@ -308,15 +309,15 @@ func TestMesh_DeadPeerRemovedAndRejoin(t *testing.T) {
 	// relay retries because the peer list is cached for up to the node TTL
 	registerFakePeer(t, pool, "node-dead", srv.URL)
 	waitFor(t, func() bool {
-		require.Nil(t, mesh.Relay(model.NewDefaultMessage("mytopic", "after rejoin")))
+		require.Nil(t, mesh.ForwardMessage(model.NewDefaultMessage("mytopic", "after rejoin")))
 		mu.Lock()
 		defer mu.Unlock()
 		return received > 1
 	})
 }
 
-func TestMesh_RelayAfterClose(t *testing.T) {
-	// A Relay racing shutdown (e.g. an in-flight publish during server Stop) must not spawn
+func TestMesh_ForwardAfterClose(t *testing.T) {
+	// A ForwardMessage racing shutdown (e.g. an in-flight publish during server Stop) must not spawn
 	// a new peer queue and worker after Close: the worker would never exit (its queue is never
 	// closed) and nothing waits for it.
 	schemaDSN := dbtest.CreateTestPostgresSchema(t)
@@ -325,7 +326,7 @@ func TestMesh_RelayAfterClose(t *testing.T) {
 	require.Nil(t, err)
 	registerFakePeer(t, pool, "node-peer", "http://127.0.0.1:1")
 	require.Nil(t, mesh.Close())
-	require.Nil(t, mesh.Relay(model.NewDefaultMessage("mytopic", "too late"))) // Dropped silently
+	require.Nil(t, mesh.ForwardMessage(model.NewDefaultMessage("mytopic", "too late"))) // Dropped silently
 	mesh.mu.Lock()
 	defer mesh.mu.Unlock()
 	require.Empty(t, mesh.queues)
@@ -418,13 +419,13 @@ func TestMesh_RouteSkipsUnsubscribedPeer(t *testing.T) {
 	rr := postState(mesh, "node-b", &apiState{Topics: &apiStateTopics{Filter: topicFilter(t, "subscribed-topic")}})
 	require.Equal(t, 200, rr.Code)
 	// A topic outside the peer's state is skipped
-	require.Nil(t, mesh.Relay(model.NewDefaultMessage("other-topic", "skipped")))
+	require.Nil(t, mesh.ForwardMessage(model.NewDefaultMessage("other-topic", "skipped")))
 	time.Sleep(300 * time.Millisecond) // Give a wrong implementation time to deliver anyway
 	mu.Lock()
 	require.Equal(t, 0, received)
 	mu.Unlock()
 	// A topic inside the peer's state is delivered
-	require.Nil(t, mesh.Relay(model.NewDefaultMessage("subscribed-topic", "delivered")))
+	require.Nil(t, mesh.ForwardMessage(model.NewDefaultMessage("subscribed-topic", "delivered")))
 	waitFor(t, func() bool {
 		mu.Lock()
 		defer mu.Unlock()
@@ -457,7 +458,7 @@ func TestMesh_RouteBroadcastsOnStaleState(t *testing.T) {
 	mesh.statesMu.Lock()
 	mesh.states["node-b"].updatedAt = time.Now().Add(-time.Hour)
 	mesh.statesMu.Unlock()
-	require.Nil(t, mesh.Relay(model.NewDefaultMessage("other-topic", "broadcast anyway")))
+	require.Nil(t, mesh.ForwardMessage(model.NewDefaultMessage("other-topic", "broadcast anyway")))
 	waitFor(t, func() bool {
 		mu.Lock()
 		defer mu.Unlock()
@@ -534,7 +535,7 @@ func TestMesh_AnnounceClosesWindow(t *testing.T) {
 		return ok
 	})
 	// Announcements merge into the baseline right away
-	meshA.AnnounceTopics([]string{"fresh-topic"})
+	meshA.BroadcastState(&State{AddedTopics: []string{"fresh-topic"}})
 	waitFor(t, func() bool {
 		meshB.statesMu.Lock()
 		defer meshB.statesMu.Unlock()
@@ -570,3 +571,20 @@ func TestMesh_StateOfDepartedPeerPruned(t *testing.T) {
 	mesh.statesMu.Unlock()
 	require.False(t, ok)
 }
+
+func TestMesh_HealthyReflectsRegistration(t *testing.T) {
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), pool, nil, nil)
+	require.Nil(t, err)
+	defer mesh.Close()
+	require.True(t, mesh.Healthy()) // Registered synchronously at construction
+	// Stale heartbeat: peers stop forwarding to this node, so it must report unhealthy
+	mesh.mu.Lock()
+	mesh.lastRegistered = time.Now().Add(-2 * mesh.conf.NodeTTL)
+	mesh.mu.Unlock()
+	require.False(t, mesh.Healthy())
+	// A successful heartbeat restores health
+	require.Nil(t, mesh.heartbeat())
+	require.True(t, mesh.Healthy())
+}

cluster/cluster_nop.go

@@ -11,14 +11,16 @@ import (
 // no special-casing in single-node mode).
 type nopCluster struct{}
 
-func (c *nopCluster) Relay(_ *model.Message) error { return nil }
+func (c *nopCluster) ForwardMessage(_ *model.Message) error { return nil }
 
 func (c *nopCluster) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
 	w.WriteHeader(http.StatusNotFound)
 }
 
-func (c *nopCluster) AnnounceTopics(_ []string) {}
+func (c *nopCluster) BroadcastState(_ *State) {}
 
 func (c *nopCluster) IsLeader() bool { return true }
 
+func (c *nopCluster) Healthy() bool { return true }
+
 func (c *nopCluster) Close() error { return nil }

cluster/cluster_test.go

@@ -67,9 +67,10 @@ func TestNop(t *testing.T) {
 	b, err := New(&Config{}, nil, nil, nil) // not enabled -> nop cluster, no database required
 	require.Nil(t, err)
 	require.IsType(t, &nopCluster{}, b)
-	require.Nil(t, b.Relay(model.NewDefaultMessage("mytopic", "hi")))
+	require.Nil(t, b.ForwardMessage(model.NewDefaultMessage("mytopic", "hi")))
 	// A single node is trivially the leader, so leader-gated jobs run without special-casing
 	require.True(t, b.IsLeader())
+	require.True(t, b.Healthy())
 	rr := httptest.NewRecorder()
 	b.ServeHTTP(rr, httptest.NewRequest("POST", MessagePath, nil))
 	require.Equal(t, 404, rr.Code)

cluster/types.go

@@ -7,6 +7,35 @@ import (
 	"heckel.io/ntfy/v2/util"
 )
 
+// Config configures the cluster. It is assembled by the server from its own config, which keeps
+// this package free of server types.
+type Config struct {
+	Enabled             bool          // Master switch; when false, New returns the nop cluster
+	NodeID              NodeID        // Stable per-node identifier; required
+	AdvertiseURL        string        // Base URL peers use to reach this node's fan-out endpoint
+	Secret              string        // Shared secret authenticating node-to-node fan-out requests
+	HeartbeatInterval   time.Duration // How often the node registry heartbeat is refreshed
+	NodeTTL             time.Duration // Registry rows older than this do not count as live peers
+	BatchLinger         time.Duration // How long messages wait in a peer queue to form a batch; 0 = send immediately
+	StateInterval       time.Duration // How often the full subscription state is pushed to peers
+	MaxMessageBytes     int64         // Upper bound for a single message on the wire (batch limits derive from this)
+	LeaderRenewInterval time.Duration // Overrides the leader lease renewal cadence; tests only, 0 = default
+}
+
+// DeliverFunc hands a message received from a peer node to this node's local subscribers. The
+// server supplies it, which inverts the dependency: this package never imports the server.
+type DeliverFunc func(m *model.Message)
+
+// State is a subscription-state delta for Cluster.BroadcastState.
+type State struct {
+	AddedTopics []string // Topics that just gained their first local subscriber on this node
+}
+
+// TopicsFunc returns the topics that currently have at least one live subscriber, computed
+// fresh on every call: membership is never tracked as a list, so topics "leave" simply by not
+// appearing in the next snapshot. The server supplies it (same inversion as DeliverFunc).
+type TopicsFunc func() []string
+
 // apiMessage is one line of a message request body (NDJSON: one message per line; a single
 // message is just a one-line body). It carries the two fields that model.Message does not
 // serialize to JSON (Sender and User), which are needed to reconstruct the visitor on the
@@ -31,7 +60,7 @@ type apiStateTopics struct {
 	Added  []string `json:"added,omitempty"`
 }
 
-// peerState is what a peer last told us about itself; Relay routes around peers whose
+// peerState is what a peer last told us about itself; ForwardMessage routes around peers whose
 // fresh state provably excludes a topic.
 type peerState struct {
 	topics    *util.BloomFilter
@@ -40,7 +69,7 @@ type peerState struct {
 
 // peerQueue is the bounded, batching send queue for a single peer, pinned to the advertise URL
 // the peer was created with: a peer re-registering under a different advertise URL is treated
-// as a replacement (reconcile retires the old queue; Relay creates a fresh one on demand).
+// as a replacement (reconcile retires the old queue; ForwardMessage creates a fresh one on demand).
 type peerQueue struct {
 	advertiseURL string
 	queue        *util.LingerQueue[[]byte] // pre-marshaled apiMessage fragments