Users() optimization to help with startup time

user/manager.go

@@ -422,33 +422,14 @@ func (a *Manager) UserByStripeCustomer(customerID string) (*User, error) {
 	return a.readUser(rows)
 }
 
-// Users returns a list of users
+// Users returns a list of users. It loads all users in a single query
+// rather than one query per user to avoid N+1 performance issues.
 func (a *Manager) Users() ([]*User, error) {
-	rows, err := a.db.Query(a.queries.selectUsernames)
+	rows, err := a.db.Query(a.queries.selectUsers)
 	if err != nil {
 		return nil, err
 	}
-	defer rows.Close()
-	usernames := make([]string, 0)
-	for rows.Next() {
-		var username string
-		if err := rows.Scan(&username); err != nil {
-			return nil, err
-		} else if err := rows.Err(); err != nil {
-			return nil, err
-		}
-		usernames = append(usernames, username)
-	}
-	rows.Close()
-	users := make([]*User, 0)
-	for _, username := range usernames {
-		user, err := a.User(username)
-		if err != nil {
-			return nil, err
-		}
-		users = append(users, user)
-	}
-	return users, nil
+	return a.readUsers(rows)
 }
 
 // UsersCount returns the number of users in the database
@@ -470,14 +451,35 @@ func (a *Manager) UsersCount() (int64, error) {
 
 func (a *Manager) readUser(rows *sql.Rows) (*User, error) {
 	defer rows.Close()
+	if !rows.Next() {
+		return nil, ErrUserNotFound
+	}
+	user, err := a.scanUser(rows)
+	if err != nil {
+		return nil, err
+	}
+	return user, nil
+}
+
+func (a *Manager) readUsers(rows *sql.Rows) ([]*User, error) {
+	defer rows.Close()
+	users := make([]*User, 0)
+	for rows.Next() {
+		user, err := a.scanUser(rows)
+		if err != nil {
+			return nil, err
+		}
+		users = append(users, user)
+	}
+	return users, nil
+}
+
+func (a *Manager) scanUser(rows *sql.Rows) (*User, error) {
 	var id, username, hash, role, prefs, syncTopic string
 	var provisioned bool
 	var stripeCustomerID, stripeSubscriptionID, stripeSubscriptionStatus, stripeSubscriptionInterval, stripeMonthlyPriceID, stripeYearlyPriceID, tierID, tierCode, tierName sql.NullString
 	var messages, emails, calls int64
 	var messagesLimit, messagesExpiryDuration, emailsLimit, callsLimit, reservationsLimit, attachmentFileSizeLimit, attachmentTotalSizeLimit, attachmentExpiryDuration, attachmentBandwidthLimit, stripeSubscriptionPaidUntil, stripeSubscriptionCancelAt, deleted sql.NullInt64
-	if !rows.Next() {
-		return nil, ErrUserNotFound
-	}
 	if err := rows.Scan(&id, &username, &hash, &role, &prefs, &syncTopic, &provisioned, &messages, &emails, &calls, &stripeCustomerID, &stripeSubscriptionID, &stripeSubscriptionStatus, &stripeSubscriptionInterval, &stripeSubscriptionPaidUntil, &stripeSubscriptionCancelAt, &deleted, &tierID, &tierCode, &tierName, &messagesLimit, &messagesExpiryDuration, &emailsLimit, &callsLimit, &reservationsLimit, &attachmentFileSizeLimit, &attachmentTotalSizeLimit, &attachmentExpiryDuration, &attachmentBandwidthLimit, &stripeMonthlyPriceID, &stripeYearlyPriceID); err != nil {
 		return nil, err
 	} else if err := rows.Err(); err != nil {
@@ -1244,6 +1246,12 @@ func (a *Manager) maybeProvisionUsersAccessAndTokens() error {
 	if !a.config.ProvisionEnabled {
 		return nil
 	}
+	// If there is nothing to provision, remove any previously provisioned items using
+	// cheap targeted queries, avoiding the expensive Users() call that loads all users.
+	if len(a.config.Users) == 0 && len(a.config.Access) == 0 && len(a.config.Tokens) == 0 {
+		return a.removeAllProvisioned()
+	}
+	// If there are provisioned users, do it the slow way
 	existingUsers, err := a.Users()
 	if err != nil {
 		return err
@@ -1269,6 +1277,23 @@ func (a *Manager) maybeProvisionUsersAccessAndTokens() error {
 	})
 }
 
+// removeAllProvisioned removes all provisioned users, access entries, and tokens. This is the fast path
+// for when there is nothing to provision, avoiding the expensive Users() call.
+func (a *Manager) removeAllProvisioned() error {
+	return db.ExecTx(a.db, func(tx *sql.Tx) error {
+		if _, err := tx.Exec(a.queries.deleteUserAccessProvisioned); err != nil {
+			return err
+		}
+		if _, err := tx.Exec(a.queries.deleteAllProvisionedTokens); err != nil {
+			return err
+		}
+		if _, err := tx.Exec(a.queries.deleteUsersProvisioned); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
 // maybeProvisionUsers checks if the users in the config are provisioned, and adds or updates them.
 // It also removes users that are provisioned, but not in the config anymore.
 func (a *Manager) maybeProvisionUsers(tx *sql.Tx, provisionUsernames []string, existingUsers []*User) error {

user/manager_postgres.go

@@ -7,6 +7,17 @@ import (
 // PostgreSQL queries
 const (
 	// User queries
+	postgresSelectUsersQuery = `
+		SELECT u.id, u.user_name, u.pass, u.role, u.prefs, u.sync_topic, u.provisioned, u.stats_messages, u.stats_emails, u.stats_calls, u.stripe_customer_id, u.stripe_subscription_id, u.stripe_subscription_status, u.stripe_subscription_interval, u.stripe_subscription_paid_until, u.stripe_subscription_cancel_at, u.deleted, t.id, t.code, t.name, t.messages_limit, t.messages_expiry_duration, t.emails_limit, t.calls_limit, t.reservations_limit, t.attachment_file_size_limit, t.attachment_total_size_limit, t.attachment_expiry_duration, t.attachment_bandwidth_limit, t.stripe_monthly_price_id, t.stripe_yearly_price_id
+		FROM "user" u
+		LEFT JOIN tier t on t.id = u.tier_id
+		ORDER BY
+			CASE u.role
+				WHEN 'admin' THEN 1
+				WHEN 'anonymous' THEN 3
+				ELSE 2
+			END, u.user_name
+	`
 	postgresSelectUserByIDQuery = `
 		SELECT u.id, u.user_name, u.pass, u.role, u.prefs, u.sync_topic, u.provisioned, u.stats_messages, u.stats_emails, u.stats_calls, u.stripe_customer_id, u.stripe_subscription_id, u.stripe_subscription_status, u.stripe_subscription_interval, u.stripe_subscription_paid_until, u.stripe_subscription_cancel_at, u.deleted, t.id, t.code, t.name, t.messages_limit, t.messages_expiry_duration, t.emails_limit, t.calls_limit, t.reservations_limit, t.attachment_file_size_limit, t.attachment_total_size_limit, t.attachment_expiry_duration, t.attachment_bandwidth_limit, t.stripe_monthly_price_id, t.stripe_yearly_price_id
 		FROM "user" u
@@ -56,6 +67,7 @@ const (
 	postgresDeleteUserQuery               = `DELETE FROM "user" WHERE user_name = $1`
 	postgresDeleteUserTierQuery           = `UPDATE "user" SET tier_id = null WHERE user_name = $1`
 	postgresDeleteUsersMarkedQuery        = `DELETE FROM "user" WHERE deleted < $1`
+	postgresDeleteUsersProvisionedQuery   = `DELETE FROM "user" WHERE provisioned = true`
 
 	// Access queries
 	postgresSelectTopicPermsQuery = `
@@ -146,13 +158,14 @@ const (
 		ON CONFLICT (user_id, token)
 		DO UPDATE SET label = excluded.label, expires = excluded.expires, provisioned = excluded.provisioned
 	`
-	postgresUpdateTokenQuery            = `UPDATE user_token SET label = $1, expires = $2 WHERE user_id = $3 AND token = $4`
-	postgresUpdateTokenLastAccessQuery  = `UPDATE user_token SET last_access = $1, last_origin = $2 WHERE token = $3`
-	postgresDeleteTokenQuery            = `DELETE FROM user_token WHERE user_id = $1 AND token = $2`
-	postgresDeleteProvisionedTokenQuery = `DELETE FROM user_token WHERE token = $1`
-	postgresDeleteAllTokenQuery         = `DELETE FROM user_token WHERE user_id = $1`
-	postgresDeleteExpiredTokensQuery    = `DELETE FROM user_token WHERE expires > 0 AND expires < $1`
-	postgresDeleteExcessTokensQuery     = `
+	postgresUpdateTokenQuery                = `UPDATE user_token SET label = $1, expires = $2 WHERE user_id = $3 AND token = $4`
+	postgresUpdateTokenLastAccessQuery      = `UPDATE user_token SET last_access = $1, last_origin = $2 WHERE token = $3`
+	postgresDeleteTokenQuery                = `DELETE FROM user_token WHERE user_id = $1 AND token = $2`
+	postgresDeleteProvisionedTokenQuery     = `DELETE FROM user_token WHERE token = $1`
+	postgresDeleteAllProvisionedTokensQuery = `DELETE FROM user_token WHERE provisioned = true`
+	postgresDeleteAllTokenQuery             = `DELETE FROM user_token WHERE user_id = $1`
+	postgresDeleteExpiredTokensQuery        = `DELETE FROM user_token WHERE expires > 0 AND expires < $1`
+	postgresDeleteExcessTokensQuery         = `
 		DELETE FROM user_token
 		WHERE user_id = $1
 		  AND (user_id, token) NOT IN (
@@ -210,6 +223,7 @@ var postgresQueries = queries{
 	selectUserByToken:            postgresSelectUserByTokenQuery,
 	selectUserByStripeCustomerID: postgresSelectUserByStripeCustomerIDQuery,
 	selectUsernames:              postgresSelectUsernamesQuery,
+	selectUsers:                  postgresSelectUsersQuery,
 	selectUserCount:              postgresSelectUserCountQuery,
 	selectUserIDFromUsername:     postgresSelectUserIDFromUsernameQuery,
 	insertUser:                   postgresInsertUserQuery,
@@ -224,6 +238,7 @@ var postgresQueries = queries{
 	deleteUser:                   postgresDeleteUserQuery,
 	deleteUserTier:               postgresDeleteUserTierQuery,
 	deleteUsersMarked:            postgresDeleteUsersMarkedQuery,
+	deleteUsersProvisioned:       postgresDeleteUsersProvisionedQuery,
 	selectTopicPerms:             postgresSelectTopicPermsQuery,
 	selectUserAllAccess:          postgresSelectUserAllAccessQuery,
 	selectUserAccess:             postgresSelectUserAccessQuery,
@@ -246,6 +261,7 @@ var postgresQueries = queries{
 	updateTokenLastAccess:        postgresUpdateTokenLastAccessQuery,
 	deleteToken:                  postgresDeleteTokenQuery,
 	deleteProvisionedToken:       postgresDeleteProvisionedTokenQuery,
+	deleteAllProvisionedTokens:   postgresDeleteAllProvisionedTokensQuery,
 	deleteAllToken:               postgresDeleteAllTokenQuery,
 	deleteExpiredTokens:          postgresDeleteExpiredTokensQuery,
 	deleteExcessTokens:           postgresDeleteExcessTokensQuery,

user/manager_sqlite.go

@@ -12,6 +12,17 @@ import (
 
 const (
 	// User queries
+	sqliteSelectUsersQuery = `
+		SELECT u.id, u.user, u.pass, u.role, u.prefs, u.sync_topic, u.provisioned, u.stats_messages, u.stats_emails, u.stats_calls, u.stripe_customer_id, u.stripe_subscription_id, u.stripe_subscription_status, u.stripe_subscription_interval, u.stripe_subscription_paid_until, u.stripe_subscription_cancel_at, deleted, t.id, t.code, t.name, t.messages_limit, t.messages_expiry_duration, t.emails_limit, t.calls_limit, t.reservations_limit, t.attachment_file_size_limit, t.attachment_total_size_limit, t.attachment_expiry_duration, t.attachment_bandwidth_limit, t.stripe_monthly_price_id, t.stripe_yearly_price_id
+		FROM user u
+		LEFT JOIN tier t on t.id = u.tier_id
+		ORDER BY
+			CASE u.role
+				WHEN 'admin' THEN 1
+				WHEN 'anonymous' THEN 3
+				ELSE 2
+			END, u.user
+	`
 	sqliteSelectUserByIDQuery = `
 		SELECT u.id, u.user, u.pass, u.role, u.prefs, u.sync_topic, u.provisioned, u.stats_messages, u.stats_emails, u.stats_calls, u.stripe_customer_id, u.stripe_subscription_id, u.stripe_subscription_status, u.stripe_subscription_interval, u.stripe_subscription_paid_until, u.stripe_subscription_cancel_at, deleted, t.id, t.code, t.name, t.messages_limit, t.messages_expiry_duration, t.emails_limit, t.calls_limit, t.reservations_limit, t.attachment_file_size_limit, t.attachment_total_size_limit, t.attachment_expiry_duration, t.attachment_bandwidth_limit, t.stripe_monthly_price_id, t.stripe_yearly_price_id
 		FROM user u
@@ -61,6 +72,7 @@ const (
 	sqliteDeleteUserQuery               = `DELETE FROM user WHERE user = ?`
 	sqliteDeleteUserTierQuery           = `UPDATE user SET tier_id = null WHERE user = ?`
 	sqliteDeleteUsersMarkedQuery        = `DELETE FROM user WHERE deleted < ?`
+	sqliteDeleteUsersProvisionedQuery   = `DELETE FROM user WHERE provisioned = 1`
 
 	// Access queries
 	sqliteSelectTopicPermsQuery = `
@@ -144,13 +156,14 @@ const (
 		ON CONFLICT (user_id, token)
 		DO UPDATE SET label = excluded.label, expires = excluded.expires, provisioned = excluded.provisioned
 	`
-	sqliteUpdateTokenQuery            = `UPDATE user_token SET label = ?, expires = ? WHERE user_id = ? AND token = ?`
-	sqliteUpdateTokenLastAccessQuery  = `UPDATE user_token SET last_access = ?, last_origin = ? WHERE token = ?`
-	sqliteDeleteTokenQuery            = `DELETE FROM user_token WHERE user_id = ? AND token = ?`
-	sqliteDeleteProvisionedTokenQuery = `DELETE FROM user_token WHERE token = ?`
-	sqliteDeleteAllTokenQuery         = `DELETE FROM user_token WHERE user_id = ?`
-	sqliteDeleteExpiredTokensQuery    = `DELETE FROM user_token WHERE expires > 0 AND expires < ?`
-	sqliteDeleteExcessTokensQuery     = `
+	sqliteUpdateTokenQuery                = `UPDATE user_token SET label = ?, expires = ? WHERE user_id = ? AND token = ?`
+	sqliteUpdateTokenLastAccessQuery      = `UPDATE user_token SET last_access = ?, last_origin = ? WHERE token = ?`
+	sqliteDeleteTokenQuery                = `DELETE FROM user_token WHERE user_id = ? AND token = ?`
+	sqliteDeleteProvisionedTokenQuery     = `DELETE FROM user_token WHERE token = ?`
+	sqliteDeleteAllProvisionedTokensQuery = `DELETE FROM user_token WHERE provisioned = 1`
+	sqliteDeleteAllTokenQuery             = `DELETE FROM user_token WHERE user_id = ?`
+	sqliteDeleteExpiredTokensQuery        = `DELETE FROM user_token WHERE expires > 0 AND expires < ?`
+	sqliteDeleteExcessTokensQuery         = `
 		DELETE FROM user_token
 		WHERE user_id = ?
 		  AND (user_id, token) NOT IN (
@@ -207,6 +220,7 @@ var sqliteQueries = queries{
 	selectUserByToken:            sqliteSelectUserByTokenQuery,
 	selectUserByStripeCustomerID: sqliteSelectUserByStripeCustomerIDQuery,
 	selectUsernames:              sqliteSelectUsernamesQuery,
+	selectUsers:                  sqliteSelectUsersQuery,
 	selectUserCount:              sqliteSelectUserCountQuery,
 	selectUserIDFromUsername:     sqliteSelectUserIDFromUsernameQuery,
 	insertUser:                   sqliteInsertUserQuery,
@@ -221,6 +235,7 @@ var sqliteQueries = queries{
 	deleteUser:                   sqliteDeleteUserQuery,
 	deleteUserTier:               sqliteDeleteUserTierQuery,
 	deleteUsersMarked:            sqliteDeleteUsersMarkedQuery,
+	deleteUsersProvisioned:       sqliteDeleteUsersProvisionedQuery,
 	selectTopicPerms:             sqliteSelectTopicPermsQuery,
 	selectUserAllAccess:          sqliteSelectUserAllAccessQuery,
 	selectUserAccess:             sqliteSelectUserAccessQuery,
@@ -243,6 +258,7 @@ var sqliteQueries = queries{
 	updateTokenLastAccess:        sqliteUpdateTokenLastAccessQuery,
 	deleteToken:                  sqliteDeleteTokenQuery,
 	deleteProvisionedToken:       sqliteDeleteProvisionedTokenQuery,
+	deleteAllProvisionedTokens:   sqliteDeleteAllProvisionedTokensQuery,
 	deleteAllToken:               sqliteDeleteAllTokenQuery,
 	deleteExpiredTokens:          sqliteDeleteExpiredTokensQuery,
 	deleteExcessTokens:           sqliteDeleteExcessTokensQuery,

user/manager_test.go

@@ -1441,6 +1441,54 @@ func TestManager_UpdateNonProvisionedUsersToProvisionedUsers(t *testing.T) {
 	})
 }
 
+func TestManager_RemoveProvisionedOnEmptyConfig(t *testing.T) {
+	forEachBackend(t, func(t *testing.T, newManager newManagerFunc) {
+		// Start with provisioned users, access, and tokens
+		conf := &Config{
+			DefaultAccess:    PermissionReadWrite,
+			ProvisionEnabled: true,
+			BcryptCost:       bcrypt.MinCost,
+			Users: []*User{
+				{Name: "provuser", Hash: "$2a$10$YLiO8U21sX1uhZamTLJXHuxgVC0Z/GKISibrKCLohPgtG7yIxSk4C", Role: RoleUser},
+			},
+			Access: map[string][]*Grant{
+				"provuser": {
+					{TopicPattern: "stats", Permission: PermissionReadWrite},
+				},
+			},
+			Tokens: map[string][]*Token{
+				"provuser": {
+					{Value: "tk_op56p8lz5bf3cxkz9je99v9oc37lo", Label: "Provisioned token"},
+				},
+			},
+		}
+		a := newTestManagerFromConfig(t, newManager, conf)
+
+		// Also add a manual (non-provisioned) user
+		require.Nil(t, a.AddUser("manualuser", "manual", RoleUser, false))
+
+		// Verify initial state
+		users, err := a.Users()
+		require.Nil(t, err)
+		require.Len(t, users, 3) // provuser, manualuser, everyone
+
+		// Re-open with empty provisioning config (simulates config change)
+		require.Nil(t, a.Close())
+		conf.Users = nil
+		conf.Access = nil
+		conf.Tokens = nil
+		a = newTestManagerFromConfig(t, newManager, conf)
+
+		// Provisioned user should be removed, manual user should remain
+		users, err = a.Users()
+		require.Nil(t, err)
+		require.Len(t, users, 2)
+		require.Equal(t, "manualuser", users[0].Name)
+		require.False(t, users[0].Provisioned)
+		require.Equal(t, "*", users[1].Name) // everyone
+	})
+}
+
 func TestToFromSQLWildcard(t *testing.T) {
 	require.Equal(t, "up%", toSQLWildcard("up*"))
 	require.Equal(t, "up\\_%", toSQLWildcard("up_*"))

user/types.go

@@ -283,6 +283,7 @@ type queries struct {
 	selectUserByToken            string
 	selectUserByStripeCustomerID string
 	selectUsernames              string
+	selectUsers                  string
 	selectUserCount              string
 	selectUserIDFromUsername     string
 	insertUser                   string
@@ -297,6 +298,7 @@ type queries struct {
 	deleteUser                   string
 	deleteUserTier               string
 	deleteUsersMarked            string
+	deleteUsersProvisioned       string
 
 	// Access queries
 	selectTopicPerms            string
@@ -323,6 +325,7 @@ type queries struct {
 	updateTokenLastAccess      string
 	deleteToken                string
 	deleteProvisionedToken     string
+	deleteAllProvisionedTokens string
 	deleteAllToken             string
 	deleteExpiredTokens        string
 	deleteExcessTokens         string