WIP: Clsuter support (cross node delivery, leader election)

message/cache.go

@@ -35,6 +35,7 @@ type queries struct {
 	selectMessagesSinceIDScheduled   string
 	selectMessagesLatest             string
 	selectMessagesDue                string
+	selectMessagesDueForUpdate       string // Postgres-only: claims due rows via FOR UPDATE SKIP LOCKED; empty for SQLite/mem
 	deleteExpiredMessages            string
 	updateMessagePublished           string
 	selectMessagesCount              string
@@ -238,6 +239,15 @@ func (c *Cache) messagesLatest(topic string) ([]*model.Message, error) {
 
 // MessagesDue returns all messages that are due for publishing
 func (c *Cache) MessagesDue() ([]*model.Message, error) {
+	// On Postgres (cluster mode), claim due rows atomically so that concurrent delayed senders
+	// on other nodes cannot pick up the same message. We SELECT ... FOR UPDATE SKIP LOCKED and
+	// mark the claimed rows published in the same transaction; each row is thus handed to exactly
+	// one node. We deliberately mark published at claim time (not after delivery) to keep the row
+	// lock short: holding a transaction open across Firebase/WebPush/email delivery would be far
+	// worse than the small at-most-once window if a node crashes between claim and delivery.
+	if c.queries.selectMessagesDueForUpdate != "" {
+		return c.claimMessagesDue()
+	}
 	rows, err := c.db.Query(c.queries.selectMessagesDue, time.Now().Unix())
 	if err != nil {
 		return nil, err
@@ -245,6 +255,32 @@ func (c *Cache) MessagesDue() ([]*model.Message, error) {
 	return readMessages(rows)
 }
 
+// claimMessagesDue is the Postgres claiming path for MessagesDue (see its comment).
+func (c *Cache) claimMessagesDue() ([]*model.Message, error) {
+	tx, err := c.db.Begin()
+	if err != nil {
+		return nil, err
+	}
+	defer tx.Rollback()
+	rows, err := tx.Query(c.queries.selectMessagesDueForUpdate, time.Now().Unix())
+	if err != nil {
+		return nil, err
+	}
+	messages, err := readMessages(rows) // reads all rows and closes them
+	if err != nil {
+		return nil, err
+	}
+	for _, m := range messages {
+		if _, err := tx.Exec(c.queries.updateMessagePublished, m.ID); err != nil {
+			return nil, err
+		}
+	}
+	if err := tx.Commit(); err != nil {
+		return nil, err
+	}
+	return messages, nil
+}
+
 // DeleteExpiredMessages deletes up to `limit` expired messages in a single query
 // and returns the number of deleted rows.
 func (c *Cache) DeleteExpiredMessages(limit int) (int64, error) {

message/cache_postgres.go

@@ -61,6 +61,13 @@ const (
 		WHERE time <= $1 AND published = FALSE
 		ORDER BY time, id
 	`
+	postgresSelectMessagesDueForUpdateQuery = `
+		SELECT mid, sequence_id, time, event, expires, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user_id, content_type, encoding
+		FROM message
+		WHERE time <= $1 AND published = FALSE
+		ORDER BY time, id
+		FOR UPDATE SKIP LOCKED
+	`
 	postgresUpdateMessagePublishedQuery = `UPDATE message SET published = TRUE WHERE mid = $1`
 	postgresSelectMessagesCountQuery    = `SELECT COUNT(*) FROM message`
 	postgresSelectTopicsQuery           = `SELECT topic FROM message GROUP BY topic`
@@ -88,6 +95,7 @@ var postgresQueries = queries{
 	selectMessagesSinceIDScheduled:   postgresSelectMessagesSinceIDIncludeScheduledQuery,
 	selectMessagesLatest:             postgresSelectMessagesLatestQuery,
 	selectMessagesDue:                postgresSelectMessagesDueQuery,
+	selectMessagesDueForUpdate:       postgresSelectMessagesDueForUpdateQuery,
 	deleteExpiredMessages:            postgresDeleteExpiredMessagesQuery,
 	updateMessagePublished:           postgresUpdateMessagePublishedQuery,
 	selectMessagesCount:              postgresSelectMessagesCountQuery,

message/cache_test.go

@@ -1,6 +1,7 @@
 package message_test
 
 import (
+	"fmt"
 	"net/netip"
 	"path/filepath"
 	"sync"
@@ -556,6 +557,43 @@ func TestStore_MarkPublished(t *testing.T) {
 	})
 }
 
+func TestStore_MessagesDue_ClaimExactlyOnce(t *testing.T) {
+	// Postgres-only: exercises "FOR UPDATE SKIP LOCKED" claiming so that concurrent delayed
+	// senders running on different cluster nodes never pick up (and deliver) the same due
+	// message twice. With the non-claiming implementation, every concurrent caller sees every
+	// due row, so this test fails; with claiming, each row is returned to exactly one caller.
+	s := newTestPostgresStore(t) // skips if NTFY_TEST_DATABASE_URL is unset
+	const n = 40
+	for i := 0; i < n; i++ {
+		m := model.NewDefaultMessage("mytopic", fmt.Sprintf("scheduled %d", i))
+		m.Time = time.Now().Add(time.Hour).Unix() // future -> stored as published=FALSE
+		require.Nil(t, s.AddMessage(m))
+		// Move the time into the past so the message is due now (but still unpublished)
+		require.Nil(t, s.UpdateMessageTime(m.ID, time.Now().Add(-time.Minute).Unix()))
+	}
+	var mu sync.Mutex
+	seen := make(map[string]int)
+	var wg sync.WaitGroup
+	for c := 0; c < 6; c++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			due, err := s.MessagesDue()
+			require.Nil(t, err)
+			mu.Lock()
+			defer mu.Unlock()
+			for _, m := range due {
+				seen[m.ID]++
+			}
+		}()
+	}
+	wg.Wait()
+	require.Len(t, seen, n) // every due message was claimed
+	for id, count := range seen {
+		require.Equalf(t, 1, count, "message %s was claimed %d times, want exactly 1", id, count)
+	}
+}
+
 func TestStore_ExpireMessages(t *testing.T) {
 	forEachBackend(t, func(t *testing.T, s *message.Cache) {
 		// Add messages to two topics