WIP: Clsuter support (cross node delivery, leader election)

cluster/cluster.go

@@ -0,0 +1,139 @@
+// Package cluster implements cross-node message delivery for a multi-node ntfy cluster. Nodes
+// register themselves in a PostgreSQL node registry (control plane) and fan published messages
+// out to each other directly over HTTP (data plane); PostgreSQL is never on the message path.
+// The single-node default is the nop cluster, which does nothing.
+package cluster
+
+import (
+	"errors"
+	"net/http"
+	"time"
+
+	"heckel.io/ntfy/v2/db"
+	"heckel.io/ntfy/v2/model"
+)
+
+// The internal peer API: every kind of node-to-node communication is a path under
+// /v1/internal/, served only on the dedicated cluster listener. Future concerns (rate limit
+// counters, stats) become new paths or new sections of the state envelope.
+const (
+	// MessagePath receives batches of published messages (NDJSON, one apiMessage per line).
+	MessagePath = "/v1/internal/message"
+	// StatePath receives peer state (JSON apiState): full subscription snapshots and
+	// incremental updates.
+	StatePath = "/v1/internal/state"
+)
+
+// NodeID identifies a cluster node; it keys the registry, the per-peer queues, and the peer
+// state table.
+//
+// Naming convention: a "node" is any cluster member in the absolute sense (identity, registry,
+// config); a "peer" is another node as seen from this one (Peers, peerQueue, peerState). A peer
+// IS a node, which is why peer values carry a NodeID.
+type NodeID string
+
+const (
+	// secretHeader carries the shared secret authenticating node-to-node fan-out requests.
+	secretHeader = "X-Cluster-Secret"
+
+	// originHeader carries the sending node's ID on fan-out requests, so a node can skip
+	// requests that carry its own broadcasts (loop prevention).
+	originHeader = "X-Cluster-Origin"
+)
+
+// Content types of the peer API: message bodies are NDJSON (one JSON message per line, matching
+// the framing of ntfy's own /topic/json subscribe stream), state bodies are plain JSON. Future
+// node-to-node request types get their own paths on the cluster listener; an old node answering
+// 404 on an unknown path keeps mixed-version clusters working during rolling deploys.
+const (
+	contentTypeNDJSON = "application/x-ndjson"
+	contentTypeJSON   = "application/json"
+)
+
+const (
+	defaultHeartbeatInterval = 3 * time.Second  // How often a node refreshes its registry heartbeat
+	defaultNodeTTL           = 30 * time.Second // A node counts as live if its heartbeat is newer than this; generous to avoid false-dead flapping (see plans)
+	defaultStateInterval     = 15 * time.Second // How often the full subscription state is pushed to peers
+
+	// DefaultBatchLinger is how long a fan-out message may wait in a peer's queue for more
+	// messages to arrive, so they are delivered as one batch. It trades up to this much
+	// cross-node latency for a bounded request rate per peer.
+	DefaultBatchLinger = 500 * time.Millisecond
+)
+
+// Config configures the cluster. It is assembled by the server from its own config, which keeps
+// this package free of server types.
+type Config struct {
+	Enabled           bool          // Master switch; when false, New returns the nop cluster
+	NodeID            NodeID        // Stable per-node identifier; required
+	AdvertiseURL      string        // Base URL peers use to reach this node's fan-out endpoint
+	Secret            string        // Shared secret authenticating node-to-node fan-out requests
+	HeartbeatInterval time.Duration // How often the node registry heartbeat is refreshed
+	NodeTTL           time.Duration // Registry rows older than this do not count as live peers
+	BatchLinger       time.Duration // How long messages wait in a peer queue to form a batch; 0 = send immediately
+	StateInterval     time.Duration // How often the full subscription state is pushed to peers
+	MaxMessageBytes   int64         // Upper bound for a single message on the wire (batch limits derive from this)
+}
+
+// DeliverFunc hands a message received from a peer node to this node's local subscribers. The
+// server supplies it, which inverts the dependency: this package never imports the server.
+type DeliverFunc func(m *model.Message)
+
+// TopicsFunc returns the topics that currently have at least one live subscriber, computed
+// fresh on every call: membership is never tracked as a list, so topics "leave" simply by not
+// appearing in the next snapshot. The server supplies it (same inversion as DeliverFunc).
+type TopicsFunc func() []string
+
+// Cluster fans published messages out to peer cluster nodes and receives their fan-out requests.
+// Local delivery to a node's own subscribers still happens inline in the server; the cluster
+// only covers the cross-node hop.
+type Cluster interface {
+	// The internal peer API (/v1/internal/*), served on the dedicated cluster listener. The
+	// nop cluster answers 404.
+	http.Handler
+	// Relay sends a locally published message on to the peer nodes that may have subscribers
+	// for its topic (all of them, when subscription knowledge is missing or stale). It is
+	// fire-and-forget and must not block the caller's request path.
+	Relay(m *model.Message) error
+	// AnnounceTopics tells peers that these topics just gained their first local subscriber,
+	// closing the routing-knowledge window to ~one round trip. Nop in single-node mode.
+	AnnounceTopics(topics []string)
+	// IsLeader reports whether this node holds the cluster leader lock. Singleton background
+	// jobs (e.g. the Firebase keepaliver) are gated on the leader.
+	IsLeader() bool
+	// TODO(T8): add Healthy() bool reflecting registration health (unhealthy when the last
+	// successful Register is older than NodeTTL, i.e. when peers stop relaying to this node),
+	// so /v1/health can pull a DB-partitioned node out of DNS rotation. Needs a fail-open
+	// answer at the health checker first: during a full DB outage ALL nodes fail to register
+	// while the mesh keeps delivering on stale peer caches, and naively pulling every node
+	// would turn a control-plane blip into a total outage.
+	// Close stops the cluster and releases its resources.
+	Close() error
+}
+
+// New creates the cluster for the given config: the nop cluster when clustering is disabled (the
+// single-node default), or the peer-mesh cluster otherwise.
+func New(conf *Config, pool *db.DB, deliver DeliverFunc, topics TopicsFunc) (Cluster, error) {
+	if !conf.Enabled {
+		return &nopCluster{}, nil
+	}
+	if pool == nil {
+		return nil, errors.New("cluster mode requires a PostgreSQL database (set database-url)")
+	}
+	if conf.AdvertiseURL == "" {
+		return nil, errors.New("cluster mode requires an advertise URL (set cluster-advertise-url)")
+	}
+	if conf.NodeID == "" {
+		return nil, errors.New("cluster mode requires a stable node ID (set cluster-node-id)")
+	}
+	if conf.HeartbeatInterval == 0 {
+		conf.HeartbeatInterval = defaultHeartbeatInterval
+	}
+	if conf.NodeTTL == 0 {
+		conf.NodeTTL = defaultNodeTTL
+	}
+	if conf.StateInterval == 0 {
+		conf.StateInterval = defaultStateInterval
+	}
+	return newMeshCluster(conf, pool, deliver, topics)
+}

cluster/cluster_load_test.go

@@ -0,0 +1,133 @@
+package cluster
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	dbtest "heckel.io/ntfy/v2/db/test"
+	"heckel.io/ntfy/v2/model"
+)
+
+// TestMesh_Soak floods the mesh with concurrent publishers and asserts exact delivery: every
+// message reaches the peer exactly once, nothing is dropped, and batching keeps the request
+// count far below the message count. Skipped unless NTFY_TEST_SOAK is set (it takes a few
+// seconds and is meant for pre-deploy verification, not the regular suite).
+func TestMesh_Soak(t *testing.T) {
+	if os.Getenv("NTFY_TEST_SOAK") == "" {
+		t.Skip("NTFY_TEST_SOAK not set")
+	}
+	// ~1000 msg/s aggregate (10x the ntfy.sh peak of ~88 msg/s): each publisher paces itself to
+	// 100 msg/s. Unthrottled publishing intentionally overruns the bounded per-peer queue (load
+	// shedding by design), so a zero-drop assertion only holds below the drain ceiling.
+	const (
+		publishers           = 10
+		messagesPerPublisher = 300
+		publishInterval      = 10 * time.Millisecond
+		total                = publishers * messagesPerPublisher
+	)
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	var mu sync.Mutex
+	received := make(map[string]int, total) // message body -> count, to catch duplicates
+	requests := 0
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		require.Nil(t, err)
+		messages, err := unmarshalMessageBody(body, 1<<20)
+		require.Nil(t, err)
+		mu.Lock()
+		requests++
+		for _, m := range messages {
+			received[m.Message]++
+		}
+		mu.Unlock()
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+	conf := newTestMeshConfig("node-a", "http://127.0.0.1:1")
+	conf.BatchLinger = 50 * time.Millisecond
+	conf.NodeTTL = time.Minute // The fake peer never heartbeats; liveness is not under test here
+	registerFakePeer(t, pool, "node-peer", srv.URL)
+	mesh, err := newMeshCluster(conf, pool, nil, nil)
+	require.Nil(t, err)
+	defer mesh.Close()
+	start := time.Now()
+	var wg sync.WaitGroup
+	for p := 0; p < publishers; p++ {
+		wg.Add(1)
+		go func(p int) {
+			defer wg.Done()
+			ticker := time.NewTicker(publishInterval)
+			defer ticker.Stop()
+			for i := 0; i < messagesPerPublisher; i++ {
+				require.Nil(t, mesh.Relay(model.NewDefaultMessage("mytopic", fmt.Sprintf("p%d-m%d", p, i))))
+				<-ticker.C
+			}
+		}(p)
+	}
+	wg.Wait()
+	waitFor(t, func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		return len(received) == total
+	})
+	elapsed := time.Since(start)
+	mu.Lock()
+	defer mu.Unlock()
+	for body, count := range received {
+		require.Equalf(t, 1, count, "message %s delivered %d times", body, count)
+	}
+	require.Less(t, requests, total/10, "expected strong batching under load")
+	t.Logf("soak: %d messages, %d requests (%.1f msgs/request), %.0f msgs/s",
+		total, requests, float64(total)/float64(requests), float64(total)/elapsed.Seconds())
+}
+
+// BenchmarkRelay measures the publish-path cost of Relay: marshal + peer lookup (cached)
+// + enqueue. The peer never drains, so enqueued fragments are dropped once the queue fills;
+// the benchmark measures the hot path, not HTTP delivery.
+func BenchmarkRelay(b *testing.B) {
+	if os.Getenv("NTFY_TEST_DATABASE_URL") == "" {
+		b.Skip("NTFY_TEST_DATABASE_URL not set")
+	}
+	schemaDSN := dbtest.CreateTestPostgresSchema(b)
+	pool := openTestPool(b, schemaDSN)
+	conf := newTestMeshConfig("node-a", "http://127.0.0.1:1")
+	conf.BatchLinger = time.Minute // Never flush; we measure enqueue only
+	mesh, err := newMeshCluster(conf, pool, nil, nil)
+	require.Nil(b, err)
+	defer mesh.Close()
+	registerFakePeer(b, pool, "node-peer", "http://127.0.0.1:1")
+	m := model.NewDefaultMessage("mytopic", "benchmark message body of typical size for a push")
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := mesh.Relay(m); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+// BenchmarkDecodeFanout measures the receive-path cost of decoding a 100-message NDJSON body.
+func BenchmarkDecodeFanout(b *testing.B) {
+	frags := make([][]byte, 100)
+	for i := range frags {
+		frag, err := marshalMessage(model.NewDefaultMessage("mytopic", fmt.Sprintf("benchmark message %d", i)))
+		require.Nil(b, err)
+		frags[i] = frag
+	}
+	body := assembleMessageBody(frags)
+	b.SetBytes(int64(len(body)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		messages, err := unmarshalMessageBody(body, 1<<20)
+		if err != nil || len(messages) != 100 {
+			b.Fatal("decode failed")
+		}
+	}
+}

cluster/cluster_mesh.go

@@ -0,0 +1,450 @@
+package cluster
+
+import (
+	"bytes"
+	"context"
+	"crypto/subtle"
+	"encoding/json"
+	"io"
+	"net/http"
+	"sync"
+	"time"
+
+	"heckel.io/ntfy/v2/cluster/registry"
+	"heckel.io/ntfy/v2/db"
+	"heckel.io/ntfy/v2/db/pg"
+	"heckel.io/ntfy/v2/log"
+	"heckel.io/ntfy/v2/metrics"
+	"heckel.io/ntfy/v2/model"
+	"heckel.io/ntfy/v2/util"
+)
+
+const (
+	meshHTTPTimeout   = 5 * time.Second
+	peerQueueSize     = 1024            // Bounded per-peer fan-out queue (drop on overflow)
+	batchMaxMessages  = 100             // Flush a batch early when it reaches this many messages
+	batchMaxBytes     = 256 * 1024      // Flush a batch early when it reaches this size
+	stateMaxBytes     = 4 * 1024 * 1024 // Upper bound for inbound state bodies (filter over ~1M topics)
+	stateFilterFPRate = 0.01            // Bloom false-positive rate; a false positive is one wasted send
+	tag               = "cluster"
+)
+
+// meshCluster fans messages out directly to peer nodes over HTTP (the data plane), using
+// PostgreSQL only as a control plane: the node_registry table for membership/discovery, and a
+// Postgres advisory lock for singleton-job leader election. Fan-out never touches the database on
+// the message path (only the cached peer list does). See plans/260715-scale-out-mesh.md.
+//
+// Each peer has its own bounded send queue and delivery worker, so a slow or wedged peer only
+// backs up (and eventually drops) its own queue and never delays delivery to healthy peers.
+type meshCluster struct {
+	conf          *Config
+	deliver       DeliverFunc
+	topics        TopicsFunc
+	registry      *registry.Registry
+	leader        *pg.Leader
+	httpClient    *http.Client
+	mux           *http.ServeMux        // The internal peer API; Cluster is an http.Handler
+	queues        map[NodeID]*peerQueue // per-peer send queues; reconciled against the registry
+	closed        bool                  // Guards against Relay spawning new workers after Close
+	states        map[NodeID]*peerState // what each peer last told us (subscription knowledge)
+	lastStatePush time.Time             // Only touched by the heartbeat goroutine
+	ctx           context.Context
+	cancel        context.CancelFunc
+	wg            sync.WaitGroup
+	mu            sync.Mutex // Protects queues and closed
+	statesMu      sync.Mutex // Protects states
+}
+
+// newMeshCluster creates the mesh cluster: it sets up the registry schema, registers this node
+// (synchronously, so it is discoverable before New returns), and starts the heartbeat loop.
+// Peer delivery workers are started lazily as peers appear in the registry.
+func newMeshCluster(conf *Config, pool *db.DB, deliver DeliverFunc, topics TopicsFunc) (*meshCluster, error) {
+	if topics == nil {
+		topics = func() []string { return nil } // No known topics; peers will broadcast to us
+	}
+	reg, err := registry.New(pool, string(conf.NodeID), conf.AdvertiseURL, conf.NodeTTL)
+	if err != nil {
+		return nil, err
+	}
+	// Register synchronously so the node is discoverable before the constructor returns; the
+	// heartbeat loop refreshes the registration from here on
+	if err := reg.Register(); err != nil {
+		return nil, err
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	c := &meshCluster{
+		conf:       conf,
+		deliver:    deliver,
+		topics:     topics,
+		registry:   reg,
+		leader:     pg.NewLeader(pool.Primary(), pg.LeaderLockKey),
+		httpClient: &http.Client{Timeout: meshHTTPTimeout},
+		queues:     make(map[NodeID]*peerQueue),
+		states:     make(map[NodeID]*peerState),
+		ctx:        ctx,
+		cancel:     cancel,
+	}
+	c.mux = http.NewServeMux()
+	c.mux.HandleFunc("POST "+MessagePath, c.authenticated(c.handleMessage))
+	c.mux.HandleFunc("POST "+StatePath, c.authenticated(c.handleState))
+	c.wg.Add(1)
+	go c.heartbeatLoop()
+	return c, nil
+}
+
+// ServeHTTP serves the internal peer API. Auth lives in the authenticated middleware, so every
+// endpoint gets the same shared-secret and origin handling.
+func (c *meshCluster) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	c.mux.ServeHTTP(w, r)
+}
+
+// authenticated wraps a peer API handler with the checks every endpoint needs: the shared
+// secret (constant-time compare, rejected before any body is read), a present origin, and the
+// origin self-skip (a request carrying this node's own traffic is acknowledged but ignored).
+func (c *meshCluster) authenticated(h func(origin NodeID, w http.ResponseWriter, r *http.Request)) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if c.conf.Secret == "" || subtle.ConstantTimeCompare([]byte(r.Header.Get(secretHeader)), []byte(c.conf.Secret)) != 1 {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		origin := NodeID(r.Header.Get(originHeader))
+		if origin == "" {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		if origin == c.conf.NodeID {
+			w.WriteHeader(http.StatusOK) // Our own traffic; nothing to do
+			return
+		}
+		h(origin, w, r)
+	}
+}
+
+// heartbeatLoop runs one heartbeat immediately (the ticker first fires a full interval after
+// startup, and a fresh node should be leader-capable and state-visible right away), then one per
+// interval until shutdown.
+func (c *meshCluster) heartbeatLoop() {
+	defer c.wg.Done()
+	ticker := time.NewTicker(c.conf.HeartbeatInterval)
+	defer ticker.Stop()
+	if err := c.heartbeat(); err != nil {
+		log.Tag(tag).Err(err).Warn("Cluster heartbeat failed")
+	}
+	for {
+		select {
+		case <-c.ctx.Done():
+			return
+		case <-ticker.C:
+			if err := c.heartbeat(); err != nil {
+				log.Tag(tag).Err(err).Warn("Cluster heartbeat failed")
+			}
+		}
+	}
+}
+
+// heartbeat is one control-plane tick: refresh this node's registry row, retry/confirm the
+// leader lock, prune long-dead registry rows (as leader), reconcile the per-peer queues, and
+// periodically push our subscription state to peers.
+//
+// A node that cannot even register itself aborts the tick: the remaining database work would
+// fail against the same database, and everything downstream degrades safely without it -- Relay
+// serves the stale peer cache on its own, and peers fall back to broadcasting to us once our
+// last pushed state expires.
+func (c *meshCluster) heartbeat() error {
+	// TODO(T8): record the last successful Register here to back the future Healthy() method
+	// (see the Cluster interface)
+	if err := c.registry.Register(); err != nil {
+		return err
+	}
+	if c.leader.TryAcquire(c.ctx) {
+		metrics.ClusterLeader.Set(1)
+		if err := c.registry.Prune(); err != nil {
+			log.Tag(tag).Err(err).Warn("Failed to prune stale nodes") // Housekeeping only; not fatal for the tick
+		}
+	} else {
+		metrics.ClusterLeader.Set(0)
+	}
+	peers, err := c.registry.Peers()
+	if err != nil {
+		return err
+	}
+	c.reconcilePeers(peers)
+	if time.Since(c.lastStatePush) >= c.conf.StateInterval {
+		c.pushState(peers)
+		c.lastStatePush = time.Now()
+	}
+	return nil
+}
+
+// reconcilePeers aligns this node's per-peer attachments with the live peer set: it retires the
+// queues (and workers) of peers that have left the registry or re-registered under a new
+// advertise URL (the retired queue's remainder was headed for a dead address anyway), and prunes
+// the stale state of departed peers. New and replacement queues are created lazily by Relay, not
+// here, so a freshly joined peer is reachable immediately.
+func (c *meshCluster) reconcilePeers(peers []*registry.Peer) {
+	metrics.ClusterPeers.Set(float64(len(peers)))
+	alive := make(map[NodeID]string, len(peers)) // node ID -> advertise URL
+	for _, p := range peers {
+		alive[NodeID(p.NodeID)] = p.AdvertiseURL
+	}
+	c.mu.Lock()
+	for nodeID, q := range c.queues {
+		if url, ok := alive[nodeID]; !ok || q.advertiseURL != url {
+			q.queue.Close() // Flushes the remainder; the worker exits when the queue is drained
+			delete(c.queues, nodeID)
+		}
+	}
+	c.mu.Unlock()
+	// Prune the state of departed peers, but only once stale: state is push-driven and can
+	// arrive before a new peer is visible in the (up to NodeTTL stale) registry view, so fresh
+	// state must survive even when its peer is not in the live set. Without this, the states of
+	// long-gone nodes would accumulate forever.
+	c.statesMu.Lock()
+	for nodeID, state := range c.states {
+		if _, ok := alive[nodeID]; !ok && time.Since(state.updatedAt) > 3*c.conf.StateInterval {
+			delete(c.states, nodeID)
+		}
+	}
+	c.statesMu.Unlock()
+}
+
+// queueFor returns the send queue for the given peer, creating it (and its delivery worker) if it
+// does not exist yet. The caller must hold c.mu.
+func (c *meshCluster) queueFor(p *registry.Peer) *peerQueue {
+	nodeID := NodeID(p.NodeID)
+	q, ok := c.queues[nodeID]
+	if ok {
+		return q
+	}
+	q = &peerQueue{
+		advertiseURL: p.AdvertiseURL,
+		queue: util.NewLingerQueue(peerQueueSize, batchMaxMessages, batchMaxBytes,
+			func(frag []byte) int { return len(frag) }, c.conf.BatchLinger),
+	}
+	c.queues[nodeID] = q
+	c.wg.Add(1)
+	go c.peerWorker(nodeID, q)
+	return q
+}
+
+// Relay enqueues the message for delivery to every live peer node that may have subscribers for
+// its topic (all of them, absent fresh knowledge). Delivery is fire-and-forget via each peer's
+// bounded batching queue; if a peer's queue is full the message is dropped for that peer
+// (subscribers reconnect and re-poll history from the database).
+func (c *meshCluster) Relay(msg *model.Message) error {
+	peers, err := c.registry.Peers()
+	if err != nil {
+		return err
+	}
+	if len(peers) == 0 {
+		return nil // Cluster of one; skip the marshal
+	}
+	frag, err := marshalMessage(msg)
+	if err != nil {
+		return err
+	}
+	metrics.ClusterMessagesRelayed.Inc()
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.closed {
+		return nil // Shutting down; the message is dropped like any other in-flight fan-out
+	}
+	for _, p := range peers {
+		// Route around peers whose fresh state provably excludes this topic; anything less
+		// certain (no state, stale state) falls back to broadcasting
+		if !c.mayNeed(NodeID(p.NodeID), msg.Topic) {
+			metrics.ClusterRouteSkipped.Inc()
+			continue
+		}
+		if !c.queueFor(p).queue.TryEnqueue(frag) {
+			metrics.ClusterQueueDropped.Inc()
+			log.Tag(tag).Warn("Fan-out queue for peer %s full, dropping message %s", p.NodeID, msg.ID)
+		}
+	}
+	return nil
+}
+
+// mayNeed reports whether the peer may have a subscriber for the topic. Conservative by
+// construction: it returns false only when a fresh state snapshot provably excludes the topic.
+// A false positive costs one wasted send; a false negative would lose a message and cannot
+// happen for topics a peer has reported (Bloom filters have no false negatives).
+func (c *meshCluster) mayNeed(peer NodeID, topic string) bool {
+	c.statesMu.Lock()
+	defer c.statesMu.Unlock()
+	state, ok := c.states[peer]
+	if !ok || time.Since(state.updatedAt) > 3*c.conf.StateInterval {
+		return true // No knowledge, or too old to trust for skipping
+	}
+	return state.topics.Contains(topic)
+}
+
+// peerWorker delivers batches of queued fan-out messages to a single peer. Batches form in the
+// peer's LingerQueue (up to BatchLinger delay, flushed early on size/count caps); the worker
+// exits when the queue is closed (peer left the registry, or mesh shutdown) and drained.
+func (c *meshCluster) peerWorker(nodeID NodeID, q *peerQueue) {
+	defer c.wg.Done()
+	for frags := range q.queue.Dequeue() {
+		c.postToPeer(nodeID, messageURL(q.advertiseURL), contentTypeNDJSON, assembleMessageBody(frags))
+		metrics.ClusterBatchesSent.Inc()
+	}
+}
+
+// postToPeer POSTs a peer API payload, authenticated with the shared cluster secret. Failures
+// are logged and counted, never retried: peer traffic is best-effort by design (messages are
+// recovered via since= replay, state via the next periodic push).
+func (c *meshCluster) postToPeer(nodeID NodeID, url, contentType string, payload []byte) {
+	req, err := http.NewRequestWithContext(c.ctx, http.MethodPost, url, bytes.NewReader(payload))
+	if err != nil {
+		metrics.ClusterSendErrors.Inc()
+		log.Tag(tag).Err(err).Warn("Failed to build request for peer %s", nodeID)
+		return
+	}
+	req.Header.Set("Content-Type", contentType)
+	req.Header.Set(secretHeader, c.conf.Secret)
+	req.Header.Set(originHeader, string(c.conf.NodeID))
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		if c.ctx.Err() == nil {
+			metrics.ClusterSendErrors.Inc()
+			log.Tag(tag).Err(err).Warn("Failed to send to peer %s (%s)", nodeID, url)
+		}
+		return
+	}
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		metrics.ClusterSendErrors.Inc()
+		log.Tag(tag).Warn("Peer %s (%s) rejected request with HTTP %d", nodeID, url, resp.StatusCode)
+	}
+}
+
+// handleMessage receives a batch of peer messages (NDJSON) and streams them to local
+// subscribers line by line, delivering each message as it is decoded.
+func (c *meshCluster) handleMessage(_ NodeID, w http.ResponseWriter, r *http.Request) {
+	// A batch can exceed its byte cap by one message, plus framing overhead
+	maxBodyBytes := int64(batchMaxBytes) + c.conf.MaxMessageBytes + 1024
+	if err := decodeMessageBody(io.LimitReader(r.Body, maxBodyBytes), int(c.conf.MaxMessageBytes), c.deliver); err != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+	w.WriteHeader(http.StatusOK)
+}
+
+// handleState receives a peer's state envelope and applies each section it carries.
+func (c *meshCluster) handleState(origin NodeID, w http.ResponseWriter, r *http.Request) {
+	body, err := io.ReadAll(io.LimitReader(r.Body, stateMaxBytes))
+	if err != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+	var state apiState
+	if err := json.Unmarshal(body, &state); err != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+	if state.Topics != nil {
+		if err := c.applyTopicState(origin, state.Topics); err != nil {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+	}
+	w.WriteHeader(http.StatusOK)
+}
+
+// applyTopicState updates what we know about a peer's subscriptions: a full snapshot replaces
+// all prior knowledge, an incremental add merges into it. Increments without a baseline are
+// ignored on purpose -- without a snapshot the peer is broadcast to anyway.
+func (c *meshCluster) applyTopicState(origin NodeID, topics *apiStateTopics) error {
+	c.statesMu.Lock()
+	defer c.statesMu.Unlock()
+	if len(topics.Filter) > 0 {
+		filter, err := util.UnmarshalBloomFilter(topics.Filter)
+		if err != nil {
+			return err
+		}
+		c.states[origin] = &peerState{topics: filter, updatedAt: time.Now()}
+		return nil
+	}
+	if state, ok := c.states[origin]; ok {
+		for _, topic := range topics.Added {
+			state.topics.Add(topic)
+		}
+		state.updatedAt = time.Now()
+	}
+	return nil
+}
+
+// pushState sends a full state snapshot to every live peer: a Bloom filter over the topics that
+// currently have local subscribers. Sent directly (not via the linger queues -- state must not
+// wait behind message batches); a lost push self-heals at the next interval. Topics without
+// subscribers disappear simply by not being in the next snapshot.
+func (c *meshCluster) pushState(peers []*registry.Peer) {
+	if len(peers) == 0 {
+		return
+	}
+	topics := c.topics()
+	filter := util.NewBloomFilter(len(topics), stateFilterFPRate)
+	for _, topic := range topics {
+		filter.Add(topic)
+	}
+	data, err := filter.MarshalBinary()
+	if err != nil {
+		return
+	}
+	body, err := json.Marshal(&apiState{Topics: &apiStateTopics{Filter: data}})
+	if err != nil {
+		return
+	}
+	for _, p := range peers {
+		go c.postToPeer(NodeID(p.NodeID), stateURL(p.AdvertiseURL), contentTypeJSON, body)
+	}
+	metrics.ClusterStatePushes.Inc()
+}
+
+// AnnounceTopics immediately tells all live peers that these topics gained their first local
+// subscriber, shrinking the window in which a publisher could wrongly skip this node from a
+// full state interval down to about one round trip.
+func (c *meshCluster) AnnounceTopics(topics []string) {
+	if len(topics) == 0 {
+		return
+	}
+	peers, err := c.registry.Peers()
+	if err != nil || len(peers) == 0 {
+		return
+	}
+	body, err := json.Marshal(&apiState{Topics: &apiStateTopics{Added: topics}})
+	if err != nil {
+		return
+	}
+	for _, p := range peers {
+		go c.postToPeer(NodeID(p.NodeID), stateURL(p.AdvertiseURL), contentTypeJSON, body)
+	}
+}
+
+// IsLeader reports whether this node currently holds singleton-job leadership.
+func (c *meshCluster) IsLeader() bool {
+	return c.leader.IsLeader()
+}
+
+// Close stops the mesh: it deregisters this node, releases leadership, stops all peer workers,
+// and waits for them to exit.
+func (c *meshCluster) Close() error {
+	c.cancel() // Stops the heartbeat loop and aborts in-flight peer deliveries
+	// Close the peer queues so their workers flush and exit; final sends are best-effort since
+	// the context is already canceled (parity with fire-and-forget delivery)
+	c.mu.Lock()
+	c.closed = true
+	for nodeID, q := range c.queues {
+		q.queue.Close()
+		delete(c.queues, nodeID)
+	}
+	c.mu.Unlock()
+	// Wait for the loops BEFORE deregistering: an in-flight heartbeat's Register would otherwise
+	// re-insert our row right after Deregister deleted it
+	c.wg.Wait()
+	if err := c.registry.Deregister(); err != nil {
+		log.Tag(tag).Err(err).Warn("Failed to deregister node")
+	}
+	c.leader.Release()
+	metrics.ClusterLeader.Set(0)
+	return nil
+}

cluster/cluster_mesh_test.go

@@ -0,0 +1,572 @@
+package cluster
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"heckel.io/ntfy/v2/cluster/registry"
+	"heckel.io/ntfy/v2/db"
+	"heckel.io/ntfy/v2/db/pg"
+	dbtest "heckel.io/ntfy/v2/db/test"
+	"heckel.io/ntfy/v2/model"
+	"heckel.io/ntfy/v2/util"
+)
+
+const (
+	testSecret = "s3cret"
+)
+
+// openTestPool opens a dedicated connection pool to the given test schema, so that each simulated
+// node has its own pool like real nodes would.
+func openTestPool(t testing.TB, dsn string) *db.DB {
+	host, err := pg.Open(dsn)
+	require.Nil(t, err)
+	d := db.New(host, nil)
+	t.Cleanup(func() { d.Close() })
+	return d
+}
+
+func newTestMeshConfig(nodeID, advertiseURL string) *Config {
+	return &Config{
+		Enabled:           true,
+		NodeID:            NodeID(nodeID),
+		AdvertiseURL:      advertiseURL,
+		Secret:            testSecret,
+		HeartbeatInterval: 100 * time.Millisecond,
+		NodeTTL:           time.Second, // Also the peer cache bound; short so fake peers registered mid-test are seen quickly
+		MaxMessageBytes:   1 << 20,
+		StateInterval:     time.Minute, // Individual tests lower this to exercise state pushes
+	}
+}
+
+// registerFakePeer registers a fake peer via the registry (creating the table if the mesh has
+// not been constructed yet): tests register fakes before the mesh boots, since its first
+// heartbeat caches the peer list. The fake never refreshes its heartbeat.
+func registerFakePeer(t testing.TB, pool *db.DB, nodeID NodeID, url string) {
+	t.Helper()
+	reg, err := registry.New(pool, string(nodeID), url, time.Minute)
+	require.Nil(t, err)
+	require.Nil(t, reg.Register())
+}
+
+func waitFor(t *testing.T, f func() bool) {
+	t.Helper()
+	for i := 0; i < 100; i++ {
+		if f() {
+			return
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+	t.Fatal("timed out waiting for condition")
+}
+
+func TestMesh_CrossNodeDelivery(t *testing.T) {
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	poolA, poolB := openTestPool(t, schemaDSN), openTestPool(t, schemaDSN)
+	var mu sync.Mutex
+	var received []*model.Message
+	var meshB *meshCluster
+	srvB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		meshB.ServeHTTP(w, r)
+	}))
+	defer srvB.Close()
+	meshB, err := newMeshCluster(newTestMeshConfig("node-b", srvB.URL), poolB, func(m *model.Message) {
+		mu.Lock()
+		defer mu.Unlock()
+		received = append(received, m)
+	}, nil)
+	require.Nil(t, err)
+	defer meshB.Close()
+	meshA, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), poolA, func(m *model.Message) {
+		t.Error("node A must not receive its own relayed message")
+	}, nil)
+	require.Nil(t, err)
+	defer meshA.Close()
+	msg := model.NewDefaultMessage("mytopic", "hello cross-node")
+	require.Nil(t, meshA.Relay(msg))
+	waitFor(t, func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		return len(received) == 1
+	})
+	mu.Lock()
+	defer mu.Unlock()
+	require.Equal(t, "mytopic", received[0].Topic)
+	require.Equal(t, "hello cross-node", received[0].Message)
+}
+
+func TestMesh_PeerAPI_Auth(t *testing.T) {
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	var delivered int
+	mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), pool, func(m *model.Message) {
+		delivered++
+	}, nil)
+	require.Nil(t, err)
+	defer mesh.Close()
+	frag, err := marshalMessage(model.NewDefaultMessage("mytopic", "hi"))
+	require.Nil(t, err)
+	payload := assembleMessageBody([][]byte{frag})
+
+	// Wrong secret -> 401, not delivered
+	rr := httptest.NewRecorder()
+	req := httptest.NewRequest("POST", MessagePath, strings.NewReader(string(payload)))
+	req.Header.Set(secretHeader, "wrong")
+	req.Header.Set(originHeader, "node-b")
+	mesh.ServeHTTP(rr, req)
+	require.Equal(t, 401, rr.Code)
+
+	// Missing secret -> 401, not delivered
+	rr = httptest.NewRecorder()
+	mesh.ServeHTTP(rr, httptest.NewRequest("POST", MessagePath, strings.NewReader(string(payload))))
+	require.Equal(t, 401, rr.Code)
+	require.Equal(t, 0, delivered)
+
+	// Missing origin -> 400, not delivered
+	rr = httptest.NewRecorder()
+	req = httptest.NewRequest("POST", MessagePath, strings.NewReader(string(payload)))
+	req.Header.Set(secretHeader, testSecret)
+	mesh.ServeHTTP(rr, req)
+	require.Equal(t, 400, rr.Code)
+	require.Equal(t, 0, delivered)
+
+	// Correct secret and origin -> 200, delivered
+	rr = httptest.NewRecorder()
+	req = httptest.NewRequest("POST", MessagePath, strings.NewReader(string(payload)))
+	req.Header.Set(secretHeader, testSecret)
+	req.Header.Set(originHeader, "node-b")
+	mesh.ServeHTTP(rr, req)
+	require.Equal(t, 200, rr.Code)
+	require.Equal(t, 1, delivered)
+}
+
+func TestMesh_PeerAPI_SelfOrigin(t *testing.T) {
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	var delivered int
+	mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), pool, func(m *model.Message) {
+		delivered++
+	}, nil)
+	require.Nil(t, err)
+	defer mesh.Close()
+
+	// A request that carries this node's own broadcasts must not be re-delivered (loop prevention)
+	frag, err := marshalMessage(model.NewDefaultMessage("mytopic", "loop"))
+	require.Nil(t, err)
+	payload := assembleMessageBody([][]byte{frag})
+	rr := httptest.NewRecorder()
+	req := httptest.NewRequest("POST", MessagePath, strings.NewReader(string(payload)))
+	req.Header.Set(secretHeader, testSecret)
+	req.Header.Set(originHeader, "node-a") // Same as the receiving node's ID
+	mesh.ServeHTTP(rr, req)
+	require.Equal(t, 200, rr.Code)
+	require.Equal(t, 0, delivered)
+}
+
+func TestMesh_SlowPeerIsolation(t *testing.T) {
+	// A wedged peer must not delay delivery to healthy peers: each peer has its own queue and
+	// delivery worker. With a shared send queue (the design this replaces), the slow peer's
+	// requests would occupy all delivery workers and starve the fast peer.
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	var mu sync.Mutex
+	fastReceived := 0 // Messages, not requests: with batching, one request can carry many
+	srvFast := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		require.Nil(t, err)
+		messages, err := unmarshalMessageBody(body, 1<<20)
+		require.Nil(t, err)
+		mu.Lock()
+		fastReceived += len(messages)
+		mu.Unlock()
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srvFast.Close()
+	release := make(chan struct{})
+	srvSlow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		<-release // Wedged until the end of the test
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srvSlow.Close()
+	defer close(release)
+	// Register the fake peers before the mesh boots; its first heartbeat caches the peer list
+	for i, url := range []string{srvFast.URL, srvSlow.URL} {
+		registerFakePeer(t, pool, NodeID(fmt.Sprintf("node-fake-%d", i)), url)
+	}
+	mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), pool, nil, nil)
+	require.Nil(t, err)
+	defer mesh.Close()
+	const n = 20
+	for i := 0; i < n; i++ {
+		require.Nil(t, mesh.Relay(model.NewDefaultMessage("mytopic", fmt.Sprintf("message %d", i))))
+	}
+	waitFor(t, func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		return fastReceived == n
+	})
+}
+
+func TestMesh_BatchCoalescing(t *testing.T) {
+	// Messages published within the linger window arrive as batches: fewer HTTP requests than
+	// messages, with nothing lost. Fails against a one-request-per-message sender.
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	var mu sync.Mutex
+	requests, messages := 0, 0
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		require.Nil(t, err)
+		decoded, err := unmarshalMessageBody(body, 1<<20)
+		require.Nil(t, err)
+		mu.Lock()
+		requests++
+		messages += len(decoded)
+		mu.Unlock()
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+	registerFakePeer(t, pool, "node-fake", srv.URL)
+	conf := newTestMeshConfig("node-a", "http://127.0.0.1:1")
+	conf.BatchLinger = 150 * time.Millisecond
+	mesh, err := newMeshCluster(conf, pool, nil, nil)
+	require.Nil(t, err)
+	defer mesh.Close()
+	const n = 20
+	for i := 0; i < n; i++ {
+		require.Nil(t, mesh.Relay(model.NewDefaultMessage("mytopic", fmt.Sprintf("message %d", i))))
+	}
+	waitFor(t, func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		return messages == n
+	})
+	mu.Lock()
+	defer mu.Unlock()
+	require.Less(t, requests, 5, "expected %d messages coalesced into few requests, got %d", n, requests)
+}
+
+func TestMesh_DeadPeerRemovedAndRejoin(t *testing.T) {
+	// A peer that dies ungracefully (no Deregister) stops refreshing its heartbeat: after the
+	// TTL it no longer counts as live (no more sends), its queue/worker are reconciled away, the
+	// leader prunes its registry row, and a re-registered peer starts receiving again.
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	var mu sync.Mutex
+	received := 0
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		require.Nil(t, err)
+		messages, err := unmarshalMessageBody(body, 1<<20)
+		require.Nil(t, err)
+		mu.Lock()
+		received += len(messages)
+		mu.Unlock()
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+	conf := newTestMeshConfig("node-a", "http://127.0.0.1:1")
+	conf.NodeTTL = 300 * time.Millisecond // Fast expiry so the test observes TTL-based removal
+	mesh, err := newMeshCluster(conf, pool, nil, nil)
+	require.Nil(t, err)
+	defer mesh.Close()
+	// The fake peer registers once and then "dies": its heartbeat is never refreshed
+	registerFakePeer(t, pool, "node-dead", srv.URL)
+	require.Nil(t, mesh.Relay(model.NewDefaultMessage("mytopic", "while alive")))
+	waitFor(t, func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		return received == 1
+	})
+	// After the TTL, the peer is no longer live: its queue is reconciled away and its registry
+	// row is pruned by the leader (this mesh is the only real node, so it holds the lock)
+	waitFor(t, func() bool {
+		mesh.mu.Lock()
+		defer mesh.mu.Unlock()
+		return len(mesh.queues) == 0
+	})
+	waitFor(t, func() bool {
+		var count int
+		require.Nil(t, pool.QueryRow(`SELECT COUNT(*) FROM node_registry WHERE node_id = 'node-dead'`).Scan(&count))
+		return count == 0
+	})
+	require.Nil(t, mesh.Relay(model.NewDefaultMessage("mytopic", "while dead")))
+	time.Sleep(250 * time.Millisecond) // Give a wrong implementation time to deliver anyway
+	mu.Lock()
+	require.Equal(t, 1, received) // Only the first message arrived
+	mu.Unlock()
+	// The peer comes back (same node ID, fresh heartbeat) and receives messages again; the
+	// relay retries because the peer list is cached for up to the node TTL
+	registerFakePeer(t, pool, "node-dead", srv.URL)
+	waitFor(t, func() bool {
+		require.Nil(t, mesh.Relay(model.NewDefaultMessage("mytopic", "after rejoin")))
+		mu.Lock()
+		defer mu.Unlock()
+		return received > 1
+	})
+}
+
+func TestMesh_RelayAfterClose(t *testing.T) {
+	// A Relay racing shutdown (e.g. an in-flight publish during server Stop) must not spawn
+	// a new peer queue and worker after Close: the worker would never exit (its queue is never
+	// closed) and nothing waits for it.
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), pool, nil, nil)
+	require.Nil(t, err)
+	registerFakePeer(t, pool, "node-peer", "http://127.0.0.1:1")
+	require.Nil(t, mesh.Close())
+	require.Nil(t, mesh.Relay(model.NewDefaultMessage("mytopic", "too late"))) // Dropped silently
+	mesh.mu.Lock()
+	defer mesh.mu.Unlock()
+	require.Empty(t, mesh.queues)
+}
+
+func TestMesh_LeaderFailover(t *testing.T) {
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	poolA, poolB := openTestPool(t, schemaDSN), openTestPool(t, schemaDSN)
+	meshA, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), poolA, nil, nil)
+	require.Nil(t, err)
+	defer meshA.Close()
+	meshB, err := newMeshCluster(newTestMeshConfig("node-b", "http://127.0.0.1:1"), poolB, nil, nil)
+	require.Nil(t, err)
+	defer meshB.Close()
+	// Exactly one node becomes leader
+	waitFor(t, func() bool {
+		return meshA.IsLeader() != meshB.IsLeader() // Exactly one
+	})
+	// The leader steps down; the follower takes over
+	leader, follower := meshA, meshB
+	if meshB.IsLeader() {
+		leader, follower = meshB, meshA
+	}
+	require.Nil(t, leader.Close())
+	waitFor(t, follower.IsLeader)
+}
+
+func TestMesh_CloseDeregisters(t *testing.T) {
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), pool, nil, nil)
+	require.Nil(t, err)
+	var count int
+	require.Nil(t, pool.QueryRow(`SELECT COUNT(*) FROM node_registry WHERE node_id = 'node-a'`).Scan(&count))
+	require.Equal(t, 1, count)
+	require.Nil(t, mesh.Close())
+	require.Nil(t, pool.QueryRow(`SELECT COUNT(*) FROM node_registry WHERE node_id = 'node-a'`).Scan(&count))
+	require.Equal(t, 0, count)
+}
+
+// postState delivers a state envelope to a mesh's peer API, as a peer would.
+func postState(c *meshCluster, origin NodeID, state *apiState) *httptest.ResponseRecorder {
+	body, err := json.Marshal(state)
+	if err != nil {
+		panic(err)
+	}
+	rr := httptest.NewRecorder()
+	req := httptest.NewRequest("POST", StatePath, bytes.NewReader(body))
+	req.Header.Set(secretHeader, testSecret)
+	req.Header.Set(originHeader, string(origin))
+	c.ServeHTTP(rr, req)
+	return rr
+}
+
+// topicFilter builds a marshaled Bloom filter over the given topics.
+func topicFilter(t *testing.T, topics ...string) []byte {
+	t.Helper()
+	filter := util.NewBloomFilter(len(topics), 0.01)
+	for _, topic := range topics {
+		filter.Add(topic)
+	}
+	data, err := filter.MarshalBinary()
+	require.Nil(t, err)
+	return data
+}
+
+func TestMesh_RouteSkipsUnsubscribedPeer(t *testing.T) {
+	// A peer whose fresh state provably excludes a topic is not contacted for it; a topic in its
+	// state is delivered as usual.
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	var mu sync.Mutex
+	received := 0
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		require.Nil(t, err)
+		messages, err := unmarshalMessageBody(body, 1<<20)
+		require.Nil(t, err)
+		mu.Lock()
+		received += len(messages)
+		mu.Unlock()
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+	registerFakePeer(t, pool, "node-b", srv.URL)
+	mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), pool, nil, nil)
+	require.Nil(t, err)
+	defer mesh.Close()
+	// node-b reports subscribers only for "subscribed-topic"
+	rr := postState(mesh, "node-b", &apiState{Topics: &apiStateTopics{Filter: topicFilter(t, "subscribed-topic")}})
+	require.Equal(t, 200, rr.Code)
+	// A topic outside the peer's state is skipped
+	require.Nil(t, mesh.Relay(model.NewDefaultMessage("other-topic", "skipped")))
+	time.Sleep(300 * time.Millisecond) // Give a wrong implementation time to deliver anyway
+	mu.Lock()
+	require.Equal(t, 0, received)
+	mu.Unlock()
+	// A topic inside the peer's state is delivered
+	require.Nil(t, mesh.Relay(model.NewDefaultMessage("subscribed-topic", "delivered")))
+	waitFor(t, func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		return received == 1
+	})
+}
+
+func TestMesh_RouteBroadcastsOnStaleState(t *testing.T) {
+	// State too old to trust cannot justify skipping: the peer is broadcast to as if unknown.
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	var mu sync.Mutex
+	received := 0
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == MessagePath { // The mesh also pushes state here; count only messages
+			mu.Lock()
+			received++
+			mu.Unlock()
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+	registerFakePeer(t, pool, "node-b", srv.URL)
+	mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), pool, nil, nil)
+	require.Nil(t, err)
+	defer mesh.Close()
+	rr := postState(mesh, "node-b", &apiState{Topics: &apiStateTopics{Filter: topicFilter(t, "subscribed-topic")}})
+	require.Equal(t, 200, rr.Code)
+	// Age the state beyond the trust window
+	mesh.statesMu.Lock()
+	mesh.states["node-b"].updatedAt = time.Now().Add(-time.Hour)
+	mesh.statesMu.Unlock()
+	require.Nil(t, mesh.Relay(model.NewDefaultMessage("other-topic", "broadcast anyway")))
+	waitFor(t, func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		return received == 1
+	})
+}
+
+func TestMesh_StatePushReplacesAndRemoves(t *testing.T) {
+	// Node A periodically pushes a full snapshot of its live topics to node B; each snapshot
+	// REPLACES B's knowledge, so topics that lost their subscribers disappear without any
+	// explicit removal protocol.
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	poolA, poolB := openTestPool(t, schemaDSN), openTestPool(t, schemaDSN)
+	var topicsMu sync.Mutex
+	topicsA := []string{"topic-1"}
+	source := func() []string {
+		topicsMu.Lock()
+		defer topicsMu.Unlock()
+		return append([]string{}, topicsA...)
+	}
+	var meshB *meshCluster
+	srvB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		meshB.ServeHTTP(w, r)
+	}))
+	defer srvB.Close()
+	meshB, err := newMeshCluster(newTestMeshConfig("node-b", srvB.URL), poolB, nil, nil)
+	require.Nil(t, err)
+	defer meshB.Close()
+	confA := newTestMeshConfig("node-a", "http://127.0.0.1:1")
+	confA.StateInterval = 200 * time.Millisecond
+	meshA, err := newMeshCluster(confA, poolA, nil, source)
+	require.Nil(t, err)
+	defer meshA.Close()
+	// B learns A's topics via the periodic push
+	knows := func(topic string) func() bool {
+		return func() bool {
+			meshB.statesMu.Lock()
+			defer meshB.statesMu.Unlock()
+			state, ok := meshB.states["node-a"]
+			return ok && state.topics.Contains(topic)
+		}
+	}
+	waitFor(t, knows("topic-1"))
+	// A's subscribers change; the next snapshot replaces the old knowledge entirely
+	topicsMu.Lock()
+	topicsA = []string{"topic-2"}
+	topicsMu.Unlock()
+	waitFor(t, knows("topic-2"))
+	waitFor(t, func() bool { return !knows("topic-1")() })
+}
+
+func TestMesh_AnnounceClosesWindow(t *testing.T) {
+	// A topic gaining its first subscriber is announced immediately, so peers learn about it
+	// without waiting for the next full state push.
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	poolA, poolB := openTestPool(t, schemaDSN), openTestPool(t, schemaDSN)
+	var meshB *meshCluster
+	srvB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		meshB.ServeHTTP(w, r)
+	}))
+	defer srvB.Close()
+	meshB, err := newMeshCluster(newTestMeshConfig("node-b", srvB.URL), poolB, nil, nil)
+	require.Nil(t, err)
+	defer meshB.Close()
+	confA := newTestMeshConfig("node-a", "http://127.0.0.1:1")
+	confA.StateInterval = 200 * time.Millisecond // One full push establishes the baseline
+	meshA, err := newMeshCluster(confA, poolA, nil, func() []string { return []string{"existing"} })
+	require.Nil(t, err)
+	defer meshA.Close()
+	waitFor(t, func() bool {
+		meshB.statesMu.Lock()
+		defer meshB.statesMu.Unlock()
+		_, ok := meshB.states["node-a"]
+		return ok
+	})
+	// Announcements merge into the baseline right away
+	meshA.AnnounceTopics([]string{"fresh-topic"})
+	waitFor(t, func() bool {
+		meshB.statesMu.Lock()
+		defer meshB.statesMu.Unlock()
+		state, ok := meshB.states["node-a"]
+		return ok && state.topics.Contains("fresh-topic")
+	})
+}
+
+func TestMesh_StateOfDepartedPeerPruned(t *testing.T) {
+	// peerState is push-driven and can arrive before the peer is visible in the registry, so it
+	// must survive reconcile while fresh -- but a departed peer's state must not leak forever:
+	// once it is both absent from the registry and stale past the trust window, it is pruned.
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	mesh, err := newMeshCluster(newTestMeshConfig("node-a", "http://127.0.0.1:1"), pool, nil, nil)
+	require.Nil(t, err)
+	defer mesh.Close()
+	rr := postState(mesh, "node-gone", &apiState{Topics: &apiStateTopics{Filter: topicFilter(t, "some-topic")}})
+	require.Equal(t, 200, rr.Code)
+	// Fresh state of an unknown peer survives reconcile (the new-node visibility window)
+	mesh.reconcilePeers(nil)
+	mesh.statesMu.Lock()
+	_, ok := mesh.states["node-gone"]
+	mesh.statesMu.Unlock()
+	require.True(t, ok)
+	// Stale state of an absent peer is pruned
+	mesh.statesMu.Lock()
+	mesh.states["node-gone"].updatedAt = time.Now().Add(-time.Hour)
+	mesh.statesMu.Unlock()
+	mesh.reconcilePeers(nil)
+	mesh.statesMu.Lock()
+	_, ok = mesh.states["node-gone"]
+	mesh.statesMu.Unlock()
+	require.False(t, ok)
+}

cluster/cluster_nop.go

@@ -0,0 +1,24 @@
+package cluster
+
+import (
+	"net/http"
+
+	"heckel.io/ntfy/v2/model"
+)
+
+// nopCluster is the single-node default: it drops all relayed messages, rejects peer API requests, and
+// reports this node as leader (a single node is trivially the leader, so leader-gated jobs need
+// no special-casing in single-node mode).
+type nopCluster struct{}
+
+func (c *nopCluster) Relay(_ *model.Message) error { return nil }
+
+func (c *nopCluster) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+	w.WriteHeader(http.StatusNotFound)
+}
+
+func (c *nopCluster) AnnounceTopics(_ []string) {}
+
+func (c *nopCluster) IsLeader() bool { return true }
+
+func (c *nopCluster) Close() error { return nil }

cluster/cluster_test.go

@@ -0,0 +1,83 @@
+package cluster
+
+import (
+	"bytes"
+	"net/http/httptest"
+	"net/netip"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"heckel.io/ntfy/v2/model"
+)
+
+func TestDeliver_RoundTrip(t *testing.T) {
+	// The fan-out body is NDJSON: one apiDeliverMessage per line, joined from pre-marshaled
+	// fragments; the origin travels in a header, not the body
+	m1 := model.NewDefaultMessage("mytopic", "my message")
+	m1.Sender = netip.MustParseAddr("1.2.3.4")
+	m1.User = "u_abc"
+	m2 := model.NewDefaultMessage("othertopic", "other message")
+	frag1, err := marshalMessage(m1)
+	require.Nil(t, err)
+	frag2, err := marshalMessage(m2)
+	require.Nil(t, err)
+	messages, err := unmarshalMessageBody(assembleMessageBody([][]byte{frag1, frag2}), 1<<20)
+	require.Nil(t, err)
+	require.Len(t, messages, 2)
+	require.Equal(t, "mytopic", messages[0].Topic)
+	require.Equal(t, "my message", messages[0].Message)
+	// Sender and User are json:"-" on model.Message; the lines must carry and reattach them
+	require.Equal(t, netip.MustParseAddr("1.2.3.4"), messages[0].Sender)
+	require.Equal(t, "u_abc", messages[0].User)
+	require.Equal(t, "othertopic", messages[1].Topic)
+	require.False(t, messages[1].Sender.IsValid())
+}
+
+func TestDeliver_SingleMessage(t *testing.T) {
+	// A single message is just a one-line body; there is no separate single-message format
+	frag, err := marshalMessage(model.NewDefaultMessage("mytopic", "hi"))
+	require.Nil(t, err)
+	messages, err := unmarshalMessageBody(assembleMessageBody([][]byte{frag}), 1<<20)
+	require.Nil(t, err)
+	require.Len(t, messages, 1)
+}
+
+func TestDeliver_MalformedLinesSkipped(t *testing.T) {
+	// Fan-out is fire-and-forget: a malformed or message-less line is skipped (and logged), the
+	// remaining lines are still delivered
+	frag, err := marshalMessage(model.NewDefaultMessage("mytopic", "good"))
+	require.Nil(t, err)
+	body := []byte("this is not json\n{\"sender\":\"1.2.3.4\"}\n" + string(frag) + "\n\n")
+	messages, err := unmarshalMessageBody(body, 1<<20)
+	require.Nil(t, err)
+	require.Len(t, messages, 1)
+	require.Equal(t, "good", messages[0].Message)
+}
+
+// unmarshalMessageBody is a test helper collecting the messages of an NDJSON message body.
+func unmarshalMessageBody(body []byte, maxLineBytes int) ([]*model.Message, error) {
+	var messages []*model.Message
+	err := decodeMessageBody(bytes.NewReader(body), maxLineBytes, func(m *model.Message) {
+		messages = append(messages, m)
+	})
+	return messages, err
+}
+
+func TestNop(t *testing.T) {
+	b, err := New(&Config{}, nil, nil, nil) // not enabled -> nop cluster, no database required
+	require.Nil(t, err)
+	require.IsType(t, &nopCluster{}, b)
+	require.Nil(t, b.Relay(model.NewDefaultMessage("mytopic", "hi")))
+	// A single node is trivially the leader, so leader-gated jobs run without special-casing
+	require.True(t, b.IsLeader())
+	rr := httptest.NewRecorder()
+	b.ServeHTTP(rr, httptest.NewRequest("POST", MessagePath, nil))
+	require.Equal(t, 404, rr.Code)
+	require.Nil(t, b.Close())
+}
+
+func TestNew_EnabledRequiresDatabase(t *testing.T) {
+	_, err := New(&Config{Enabled: true, Secret: "secret"}, nil, nil, nil)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "database")
+}

cluster/registry/registry.go

@@ -0,0 +1,149 @@
+// Package registry implements cluster membership: each node upserts its own row into the
+// node_registry table with a fresh heartbeat, and discovers its peers by reading the other
+// fresh rows. Node IDs are plain strings here; the cluster package layers its NodeID type on
+// top.
+package registry
+
+import (
+	"sync"
+	"time"
+
+	"heckel.io/ntfy/v2/db"
+	"heckel.io/ntfy/v2/db/schema"
+)
+
+// Registry queries
+const (
+	upsertNodeQuery = `
+		INSERT INTO node_registry (node_id, advertise_url, last_heartbeat)
+		VALUES ($1, $2, $3)
+		ON CONFLICT (node_id) DO UPDATE SET advertise_url = EXCLUDED.advertise_url, last_heartbeat = EXCLUDED.last_heartbeat
+	`
+	selectPeersQuery     = `SELECT node_id, advertise_url FROM node_registry WHERE last_heartbeat >= $1 AND node_id != $2`
+	pruneStaleNodesQuery = `DELETE FROM node_registry WHERE last_heartbeat < $1`
+	deleteNodeQuery      = `DELETE FROM node_registry WHERE node_id = $1`
+)
+
+// Schema version and queries
+
+const (
+	schemaVersion  = 1
+	schemaStoreKey = "node_registry"
+)
+
+var (
+	createTable = schema.AsMigrateFunc(`
+		CREATE TABLE IF NOT EXISTS node_registry (
+			node_id        TEXT PRIMARY KEY,
+			advertise_url  TEXT NOT NULL,
+			last_heartbeat BIGINT NOT NULL
+		)
+	`)
+)
+
+// Peer is a live remote node as read from the registry.
+type Peer struct {
+	NodeID       string
+	AdvertiseURL string
+}
+
+// Registry is the node membership table (control plane): each node upserts its own row with a
+// fresh heartbeat every few seconds, and peers are the other rows with a heartbeat newer than
+// the TTL. Stale rows are pruned by the leader. The TTL bounds membership staleness in BOTH
+// directions: how long a silent node still counts as live, and how long the cached peer list is
+// served before a re-read -- so a new node may take up to a TTL to become visible.
+type Registry struct {
+	pool         *db.DB
+	nodeID       string
+	advertiseURL string
+	ttl          time.Duration
+	peers        []*Peer // cached peer list
+	peersFetched time.Time
+	mu           sync.Mutex // Protects peers and peersFetched
+}
+
+// New creates or migrates the registry schema and returns this node's membership handle. It
+// does NOT register the node: joining the cluster is an explicit Register call, owned by the
+// caller, so read-only uses of the registry stay side-effect free.
+func New(pool *db.DB, nodeID, advertiseURL string, ttl time.Duration) (*Registry, error) {
+	if err := schema.Migrate(pool.Primary(), schema.Postgres, schemaStoreKey, schemaVersion, createTable, nil); err != nil {
+		return nil, err
+	}
+	return &Registry{
+		pool:         pool,
+		nodeID:       nodeID,
+		advertiseURL: advertiseURL,
+		ttl:          ttl,
+	}, nil
+}
+
+// Register upserts this node into the registry with a fresh heartbeat. It is a pure write: it
+// does not touch the peer cache, because our own row is excluded from Peers() anyway.
+func (r *Registry) Register() error {
+	_, err := r.pool.Exec(upsertNodeQuery, r.nodeID, r.advertiseURL, time.Now().Unix())
+	return err
+}
+
+// Peers returns the current set of live peer nodes (all registry rows with a fresh heartbeat,
+// excluding this node), cached for the TTL.
+func (r *Registry) Peers() ([]*Peer, error) {
+	r.mu.Lock()
+	if r.peers != nil && time.Since(r.peersFetched) < r.ttl {
+		peers := r.peers
+		r.mu.Unlock()
+		return peers, nil
+	}
+	r.mu.Unlock()
+	peers, err := r.queryPeers()
+	if err != nil {
+		// Serve the last-known peer list during database hiccups: fan-out keeps flowing to
+		// known peers instead of erroring (and logging) once per published message for the
+		// duration of the outage. Dead peers in the stale list only cost failed sends.
+		r.mu.Lock()
+		defer r.mu.Unlock()
+		if r.peers != nil {
+			return r.peers, nil
+		}
+		return nil, err
+	}
+	r.mu.Lock()
+	r.peers = peers
+	r.peersFetched = time.Now()
+	r.mu.Unlock()
+	return peers, nil
+}
+
+// Prune deletes registry rows whose heartbeat is long expired. Only the leader calls this; the
+// grace period of 3x the TTL avoids deleting rows of nodes that are merely slow to heartbeat.
+func (r *Registry) Prune() error {
+	_, err := r.pool.Exec(pruneStaleNodesQuery, time.Now().Add(-3*r.ttl).Unix())
+	return err
+}
+
+// Deregister deletes this node's registry row; called on shutdown.
+func (r *Registry) Deregister() error {
+	_, err := r.pool.Exec(deleteNodeQuery, r.nodeID)
+	return err
+}
+
+// queryPeers reads the current live peer set from the registry table.
+func (r *Registry) queryPeers() ([]*Peer, error) {
+	cutoff := time.Now().Add(-r.ttl).Unix()
+	rows, err := r.pool.Query(selectPeersQuery, cutoff, r.nodeID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	peers := make([]*Peer, 0)
+	for rows.Next() {
+		p := &Peer{}
+		if err := rows.Scan(&p.NodeID, &p.AdvertiseURL); err != nil {
+			return nil, err
+		}
+		peers = append(peers, p)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return peers, nil
+}

cluster/registry/registry_test.go

@@ -0,0 +1,222 @@
+package registry
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"heckel.io/ntfy/v2/db"
+	"heckel.io/ntfy/v2/db/pg"
+	dbtest "heckel.io/ntfy/v2/db/test"
+)
+
+func openTestPool(t *testing.T, dsn string) *db.DB {
+	t.Helper()
+	host, err := pg.Open(dsn)
+	require.Nil(t, err)
+	d := db.New(host, nil)
+	t.Cleanup(func() { d.Close() })
+	return d
+}
+
+func TestRegistry_NewDoesNotRegister(t *testing.T) {
+	// New only sets up the schema and the identity handle; joining the cluster is an explicit
+	// Register call, owned by the caller (the mesh registers synchronously at construction).
+	// This keeps read-only uses (ops tooling, future admin endpoints) side-effect free.
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	r1, err := New(pool, "node-1", "http://10.0.0.1:2587", time.Minute)
+	require.Nil(t, err)
+	require.Equal(t, 0, countRows(t, pool, "node-1"))
+	require.Nil(t, r1.Register())
+	require.Equal(t, 1, countRows(t, pool, "node-1"))
+}
+
+func TestRegistry_RegisterAndPeers(t *testing.T) {
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	r1, err := New(pool, "node-1", "http://10.0.0.1:2587", time.Minute)
+	require.Nil(t, err)
+	require.Nil(t, r1.Register())
+	r2, err := New(pool, "node-2", "http://10.0.0.2:2587", time.Minute)
+	require.Nil(t, err)
+	require.Nil(t, r2.Register())
+	// Each node sees the other, never itself
+	peers, err := r1.Peers()
+	require.Nil(t, err)
+	require.Len(t, peers, 1)
+	require.Equal(t, "node-2", peers[0].NodeID)
+	require.Equal(t, "http://10.0.0.2:2587", peers[0].AdvertiseURL)
+	peers, err = r2.Peers()
+	require.Nil(t, err)
+	require.Len(t, peers, 1)
+	require.Equal(t, "node-1", peers[0].NodeID)
+}
+
+func TestRegistry_ReRegisterUpdatesAdvertiseURL(t *testing.T) {
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	r1, err := New(pool, "node-1", "http://10.0.0.1:2587", time.Minute)
+	require.Nil(t, err)
+	// The same node comes back under a new address; the upsert replaces the row
+	old, err := New(pool, "node-2", "http://old:2587", time.Minute)
+	require.Nil(t, err)
+	require.Nil(t, old.Register())
+	renewed, err := New(pool, "node-2", "http://new:2587", time.Minute)
+	require.Nil(t, err)
+	require.Nil(t, renewed.Register())
+	expireCache(r1)
+	peers, err := r1.Peers()
+	require.Nil(t, err)
+	require.Len(t, peers, 1)
+	require.Equal(t, "http://new:2587", peers[0].AdvertiseURL)
+}
+
+func TestRegistry_PeersCachedForTTL(t *testing.T) {
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	r1, err := New(pool, "node-1", "http://10.0.0.1:2587", time.Minute)
+	require.Nil(t, err)
+	peers, err := r1.Peers()
+	require.Nil(t, err)
+	require.Empty(t, peers)
+	// A node joining after the cache was populated is invisible until the cache expires
+	r2, err := New(pool, "node-2", "http://10.0.0.2:2587", time.Minute)
+	require.Nil(t, err)
+	require.Nil(t, r2.Register())
+	peers, err = r1.Peers()
+	require.Nil(t, err)
+	require.Empty(t, peers)
+	expireCache(r1)
+	peers, err = r1.Peers()
+	require.Nil(t, err)
+	require.Len(t, peers, 1)
+}
+
+func TestRegistry_TTLExcludesSilentNodes(t *testing.T) {
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	r1, err := New(pool, "node-1", "http://10.0.0.1:2587", time.Minute)
+	require.Nil(t, err)
+	// A node whose heartbeat is older than the TTL does not count as live
+	_, err = pool.Exec(upsertNodeQuery, "node-silent", "http://10.0.0.9:2587", time.Now().Add(-2*time.Minute).Unix())
+	require.Nil(t, err)
+	peers, err := r1.Peers()
+	require.Nil(t, err)
+	require.Empty(t, peers)
+}
+
+func TestRegistry_PruneDeletesLongDeadOnly(t *testing.T) {
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	r1, err := New(pool, "node-1", "http://10.0.0.1:2587", time.Minute)
+	require.Nil(t, err)
+	// One node beyond the 3x TTL grace period, one merely stale
+	_, err = pool.Exec(upsertNodeQuery, "node-long-dead", "http://10.0.0.8:2587", time.Now().Add(-4*time.Minute).Unix())
+	require.Nil(t, err)
+	_, err = pool.Exec(upsertNodeQuery, "node-slow", "http://10.0.0.9:2587", time.Now().Add(-2*time.Minute).Unix())
+	require.Nil(t, err)
+	require.Nil(t, r1.Prune())
+	require.Equal(t, 0, countRows(t, pool, "node-long-dead"))
+	require.Equal(t, 1, countRows(t, pool, "node-slow")) // Slow, not dead: kept
+}
+
+func TestRegistry_Deregister(t *testing.T) {
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	r1, err := New(pool, "node-1", "http://10.0.0.1:2587", time.Minute)
+	require.Nil(t, err)
+	require.Nil(t, r1.Register())
+	require.Equal(t, 1, countRows(t, pool, "node-1"))
+	require.Nil(t, r1.Deregister())
+	require.Equal(t, 0, countRows(t, pool, "node-1"))
+}
+
+func TestRegistry_PeersStaleCacheOnError(t *testing.T) {
+	// During a database hiccup, Peers serves the last-known peer list instead of erroring:
+	// fan-out keeps flowing to known peers, and the publish path does not log a warning per
+	// message for the duration of the outage.
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	r1, err := New(pool, "node-1", "http://10.0.0.1:2587", time.Minute)
+	require.Nil(t, err)
+	r2, err := New(pool, "node-2", "http://10.0.0.2:2587", time.Minute)
+	require.Nil(t, err)
+	require.Nil(t, r2.Register())
+	peers, err := r1.Peers()
+	require.Nil(t, err)
+	require.Len(t, peers, 1)
+	// Expire the cache and break the database; the stale list must still be served
+	expireCache(r1)
+	require.Nil(t, pool.Close())
+	peers, err = r1.Peers()
+	require.Nil(t, err)
+	require.Len(t, peers, 1)
+	require.Equal(t, "node-2", peers[0].NodeID)
+}
+
+func TestRegistry_ConcurrentCreate(t *testing.T) {
+	// Multiple nodes cold-booting on a fresh database must not race on table creation: CREATE
+	// TABLE IF NOT EXISTS is not atomic in PostgreSQL, so creation is serialized via an advisory
+	// lock. Without it, this test fails sporadically with a duplicate-key error on pg_class.
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	const n = 8
+	errs := make(chan error, n)
+	for i := 0; i < n; i++ {
+		go func(i int) {
+			pool, err := pg.Open(schemaDSN)
+			if err != nil {
+				errs <- err
+				return
+			}
+			defer pool.DB.Close()
+			_, err = New(db.New(pool, nil), fmt.Sprintf("node-%d", i), "http://127.0.0.1:1", time.Second)
+			errs <- err
+		}(i)
+	}
+	for i := 0; i < n; i++ {
+		require.Nil(t, <-errs)
+	}
+}
+
+func TestRegistry_SchemaVersionWritten(t *testing.T) {
+	// The registry participates in the shared schema_version framework like every other store,
+	// so future table changes can be applied as migrations.
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	_, err := New(pool, "node-1", "http://10.0.0.1:2587", time.Minute)
+	require.Nil(t, err)
+	var version int
+	require.Nil(t, pool.QueryRow(`SELECT version FROM schema_version WHERE store = $1`, schemaStoreKey).Scan(&version))
+	require.Equal(t, schemaVersion, version)
+	// Setup is idempotent: a second node boots against the migrated schema
+	_, err = New(pool, "node-2", "http://10.0.0.2:2587", time.Minute)
+	require.Nil(t, err)
+}
+
+func TestRegistry_SchemaVersionFromTheFuture(t *testing.T) {
+	// A node running older code must refuse to touch a schema migrated by newer code
+	schemaDSN := dbtest.CreateTestPostgresSchema(t)
+	pool := openTestPool(t, schemaDSN)
+	_, err := New(pool, "node-1", "http://10.0.0.1:2587", time.Minute)
+	require.Nil(t, err)
+	_, err = pool.Exec(`UPDATE schema_version SET version = 99 WHERE store = $1`, schemaStoreKey)
+	require.Nil(t, err)
+	_, err = New(pool, "node-2", "http://10.0.0.2:2587", time.Minute)
+	require.Error(t, err)
+}
+
+// expireCache forces the next Peers() call to re-read the registry table.
+func expireCache(r *Registry) {
+	r.mu.Lock()
+	r.peersFetched = time.Time{}
+	r.mu.Unlock()
+}
+
+func countRows(t *testing.T, pool *db.DB, nodeID string) int {
+	t.Helper()
+	var count int
+	require.Nil(t, pool.QueryRow(`SELECT COUNT(*) FROM node_registry WHERE node_id = $1`, nodeID).Scan(&count))
+	return count
+}

cluster/types.go

@@ -0,0 +1,47 @@
+package cluster
+
+import (
+	"time"
+
+	"heckel.io/ntfy/v2/model"
+	"heckel.io/ntfy/v2/util"
+)
+
+// apiMessage is one line of a message request body (NDJSON: one message per line; a single
+// message is just a one-line body). It carries the two fields that model.Message does not
+// serialize to JSON (Sender and User), which are needed to reconstruct the visitor on the
+// receiving node. The origin node travels in a request header, not in the body.
+type apiMessage struct {
+	Sender  string         `json:"sender,omitempty"`
+	User    string         `json:"user,omitempty"`
+	Message *model.Message `json:"message"`
+}
+
+// apiState is the peer state-exchange envelope. Each concern is an optional section; future
+// concerns (rate limit counters, stats) become siblings of Topics.
+type apiState struct {
+	Topics *apiStateTopics `json:"topics,omitempty"`
+}
+
+// apiStateTopics carries a peer's subscription knowledge: either a full snapshot (Filter, a
+// marshaled Bloom filter over the topics with live subscribers) replacing all prior knowledge,
+// or an incremental update (Added) merged into it.
+type apiStateTopics struct {
+	Filter []byte   `json:"filter,omitempty"`
+	Added  []string `json:"added,omitempty"`
+}
+
+// peerState is what a peer last told us about itself; Relay routes around peers whose
+// fresh state provably excludes a topic.
+type peerState struct {
+	topics    *util.BloomFilter
+	updatedAt time.Time
+}
+
+// peerQueue is the bounded, batching send queue for a single peer, pinned to the advertise URL
+// the peer was created with: a peer re-registering under a different advertise URL is treated
+// as a replacement (reconcile retires the old queue; Relay creates a fresh one on demand).
+type peerQueue struct {
+	advertiseURL string
+	queue        *util.LingerQueue[[]byte] // pre-marshaled apiMessage fragments
+}

cluster/util.go

@@ -0,0 +1,69 @@
+package cluster
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"io"
+	"net/netip"
+	"strings"
+
+	"heckel.io/ntfy/v2/log"
+	"heckel.io/ntfy/v2/model"
+)
+
+// messageURL derives the peer's message endpoint URL from its advertise URL.
+func messageURL(advertiseURL string) string {
+	return strings.TrimRight(advertiseURL, "/") + MessagePath
+}
+
+// stateURL derives the peer's state endpoint URL from its advertise URL.
+func stateURL(advertiseURL string) string {
+	return strings.TrimRight(advertiseURL, "/") + StatePath
+}
+
+// marshalMessage serializes one message and its non-JSON fields (Sender, User) as an
+// apiMessage line. Lines are marshaled once per publish and shared across all per-peer
+// queues; assembleMessageBody joins them without re-marshaling.
+func marshalMessage(m *model.Message) ([]byte, error) {
+	apiMsg := &apiMessage{User: m.User, Message: m}
+	if m.Sender.IsValid() {
+		apiMsg.Sender = m.Sender.String()
+	}
+	return json.Marshal(apiMsg)
+}
+
+// assembleMessageBody builds an NDJSON fan-out request body from pre-marshaled apiMessage
+// lines, avoiding a second JSON marshal of the messages.
+func assembleMessageBody(frags [][]byte) []byte {
+	return append(bytes.Join(frags, []byte("\n")), '\n')
+}
+
+// decodeMessageBody reads NDJSON apiMessage lines from r, reattaches the non-JSON fields
+// (Sender, User) onto each message, and hands them to deliver. Malformed or message-less lines
+// are skipped and logged, not fatal: fan-out is fire-and-forget, so the valid remainder of a
+// request is still delivered. It returns an error only for stream-level failures (e.g. a line
+// exceeding maxLineBytes).
+func decodeMessageBody(r io.Reader, maxLineBytes int, deliver DeliverFunc) error {
+	scanner := bufio.NewScanner(r)
+	scanner.Buffer(make([]byte, 64*1024), maxLineBytes)
+	for scanner.Scan() {
+		line := bytes.TrimSpace(scanner.Bytes())
+		if len(line) == 0 {
+			continue
+		}
+		var apiMsg apiMessage
+		if err := json.Unmarshal(line, &apiMsg); err != nil || apiMsg.Message == nil {
+			log.Tag(tag).Warn("Skipping malformed fan-out line")
+			continue
+		}
+		apiMsg.Message.User = apiMsg.User
+		if apiMsg.Sender != "" {
+			if addr, err := netip.ParseAddr(apiMsg.Sender); err == nil {
+				apiMsg.Message.Sender = addr
+			}
+		}
+		deliver(apiMsg.Message)
+	}
+	return scanner.Err()
+}