ntfy/cluster/cluster.go

// Package cluster implements cross-node message delivery for a multi-node ntfy cluster. Nodes
// register themselves in a PostgreSQL node registry (control plane) and fan published messages
// out to each other directly over HTTP (data plane); PostgreSQL is never on the message path.
// The single-node default is the nop cluster, which does nothing.
package cluster

import (
	"errors"
	"net/http"
	"time"

	"heckel.io/ntfy/v2/db"
	"heckel.io/ntfy/v2/model"
)

// The internal peer API: every kind of node-to-node communication is a path under
// /v1/internal/, served only on the dedicated cluster listener. Future concerns (rate limit
// counters, stats) become new paths or new sections of the state envelope.
const (
	// MessagePath receives batches of published messages (NDJSON, one apiMessage per line).
	MessagePath = "/v1/internal/message"
	// StatePath receives peer state (JSON apiState): full subscription snapshots and
	// incremental updates.
	StatePath = "/v1/internal/state"
)

// NodeID identifies a cluster node; it keys the registry, the per-peer queues, and the peer
// state table.
//
// Naming convention: a "node" is any cluster member in the absolute sense (identity, registry,
// config); a "peer" is another node as seen from this one (Peers, peerQueue, peerState). A peer
// IS a node, which is why peer values carry a NodeID.
type NodeID string

const (
	// secretHeader carries the shared secret authenticating node-to-node fan-out requests.
	secretHeader = "X-Cluster-Secret"

	// originHeader carries the sending node's ID on fan-out requests, so a node can skip
	// requests that carry its own broadcasts (loop prevention).
	originHeader = "X-Cluster-Origin"
)

// Content types of the peer API: message bodies are NDJSON (one JSON message per line, matching
// the framing of ntfy's own /topic/json subscribe stream), state bodies are plain JSON. Future
// node-to-node request types get their own paths on the cluster listener; an old node answering
// 404 on an unknown path keeps mixed-version clusters working during rolling deploys.
const (
	contentTypeNDJSON = "application/x-ndjson"
	contentTypeJSON   = "application/json"
)

const (
	defaultHeartbeatInterval = 3 * time.Second  // How often a node refreshes its registry heartbeat
	defaultNodeTTL           = 30 * time.Second // A node counts as live if its heartbeat is newer than this; generous to avoid false-dead flapping (see plans)
	defaultStateInterval     = 15 * time.Second // How often the full subscription state is pushed to peers

	// DefaultBatchLinger is how long a fan-out message may wait in a peer's queue for more
	// messages to arrive, so they are delivered as one batch. It trades up to this much
	// cross-node latency for a bounded request rate per peer.
	DefaultBatchLinger = 500 * time.Millisecond
)

// Config configures the cluster. It is assembled by the server from its own config, which keeps
// this package free of server types.
type Config struct {
	Enabled           bool          // Master switch; when false, New returns the nop cluster
	NodeID            NodeID        // Stable per-node identifier; required
	AdvertiseURL      string        // Base URL peers use to reach this node's fan-out endpoint
	Secret            string        // Shared secret authenticating node-to-node fan-out requests
	HeartbeatInterval time.Duration // How often the node registry heartbeat is refreshed
	NodeTTL           time.Duration // Registry rows older than this do not count as live peers
	BatchLinger       time.Duration // How long messages wait in a peer queue to form a batch; 0 = send immediately
	StateInterval     time.Duration // How often the full subscription state is pushed to peers
	MaxMessageBytes   int64         // Upper bound for a single message on the wire (batch limits derive from this)
}

// DeliverFunc hands a message received from a peer node to this node's local subscribers. The
// server supplies it, which inverts the dependency: this package never imports the server.
type DeliverFunc func(m *model.Message)

// TopicsFunc returns the topics that currently have at least one live subscriber, computed
// fresh on every call: membership is never tracked as a list, so topics "leave" simply by not
// appearing in the next snapshot. The server supplies it (same inversion as DeliverFunc).
type TopicsFunc func() []string

// Cluster fans published messages out to peer cluster nodes and receives their fan-out requests.
// Local delivery to a node's own subscribers still happens inline in the server; the cluster
// only covers the cross-node hop.
type Cluster interface {
	// The internal peer API (/v1/internal/*), served on the dedicated cluster listener. The
	// nop cluster answers 404.
	http.Handler
	// Relay sends a locally published message on to the peer nodes that may have subscribers
	// for its topic (all of them, when subscription knowledge is missing or stale). It is
	// fire-and-forget and must not block the caller's request path.
	Relay(m *model.Message) error
	// AnnounceTopics tells peers that these topics just gained their first local subscriber,
	// closing the routing-knowledge window to ~one round trip. Nop in single-node mode.
	AnnounceTopics(topics []string)
	// IsLeader reports whether this node holds the cluster leader lock. Singleton background
	// jobs (e.g. the Firebase keepaliver) are gated on the leader.
	IsLeader() bool
	// TODO(T8): add Healthy() bool reflecting registration health (unhealthy when the last
	// successful Register is older than NodeTTL, i.e. when peers stop relaying to this node),
	// so /v1/health can pull a DB-partitioned node out of DNS rotation. Needs a fail-open
	// answer at the health checker first: during a full DB outage ALL nodes fail to register
	// while the mesh keeps delivering on stale peer caches, and naively pulling every node
	// would turn a control-plane blip into a total outage.
	// Close stops the cluster and releases its resources.
	Close() error
}

// New creates the cluster for the given config: the nop cluster when clustering is disabled (the
// single-node default), or the peer-mesh cluster otherwise.
func New(conf *Config, pool *db.DB, deliver DeliverFunc, topics TopicsFunc) (Cluster, error) {
	if !conf.Enabled {
		return &nopCluster{}, nil
	}
	if pool == nil {
		return nil, errors.New("cluster mode requires a PostgreSQL database (set database-url)")
	}
	if conf.AdvertiseURL == "" {
		return nil, errors.New("cluster mode requires an advertise URL (set cluster-advertise-url)")
	}
	if conf.NodeID == "" {
		return nil, errors.New("cluster mode requires a stable node ID (set cluster-node-id)")
	}
	if conf.HeartbeatInterval == 0 {
		conf.HeartbeatInterval = defaultHeartbeatInterval
	}
	if conf.NodeTTL == 0 {
		conf.NodeTTL = defaultNodeTTL
	}
	if conf.StateInterval == 0 {
		conf.StateInterval = defaultStateInterval
	}
	return newMeshCluster(conf, pool, deliver, topics)
}