fix(sdk): include delayed messages in MAM catch-up cursor to prevent backward queries

Author: mremond

apps/fluux/src-tauri/tauri.conf.json

@@ -68,7 +68,7 @@
     ],
     "macOS": {
       "minimumSystemVersion": "10.13",
-      "bundleVersion": "e40b1fa",
+      "bundleVersion": "12987ef",
       "entitlements": "Entitlements.plist",
       "signingIdentity": null
     },

packages/fluux-sdk/src/core/chatSideEffects.test.ts

@@ -235,6 +235,74 @@ describe('setupChatSideEffects', () => {
     })
   })
 
+  describe('catch-up with delayed messages', () => {
+    it('should use forward query from newest delayed message (not fall back to backward)', async () => {
+      // This test validates the fix for GitHub issue #135:
+      // Messages sent from another client while Fluux is closed were not shown
+      // because the catch-up cursor skipped delayed messages, causing a backward
+      // query whose prepend-based merge put newer messages at the wrong position.
+      const delayedTimestamp = new Date('2026-02-15T10:00:00Z')
+
+      connectionStore.getState().setServerInfo({
+        identities: [],
+        domain: 'example.com',
+        features: [NS_MAM],
+      })
+
+      chatStore.getState().addConversation({
+        id: 'contact@example.com',
+        name: 'contact@example.com',
+        type: 'chat',
+        lastMessage: undefined,
+        unreadCount: 0,
+      })
+
+      chatStore.getState().setActiveConversation('contact@example.com')
+
+      // Mock loadMessagesFromCache to populate with ONLY delayed messages
+      // (simulates a conversation populated entirely via previous MAM catch-ups)
+      const delayedMsg = {
+        type: 'chat' as const,
+        id: 'delayed-msg-1',
+        conversationId: 'contact@example.com',
+        from: 'contact@example.com',
+        body: 'Previous MAM message',
+        timestamp: delayedTimestamp,
+        isOutgoing: false,
+        isDelayed: true,
+      }
+      const loadSpy = vi.spyOn(chatStore.getState(), 'loadMessagesFromCache')
+        .mockImplementation(async (conversationId: string) => {
+          chatStore.getState().addMessage({
+            ...delayedMsg,
+            conversationId,
+          })
+          return [delayedMsg]
+        })
+
+      connectionStore.getState().setStatus('disconnected')
+      cleanup = setupChatSideEffects(mockClient)
+
+      simulateFreshSession(mockClient)
+
+      await vi.waitFor(() => {
+        expect(mockClient.chat.queryMAM).toHaveBeenCalledWith(
+          expect.objectContaining({
+            with: 'contact@example.com',
+            start: expect.any(String),
+          })
+        )
+      })
+
+      // Verify it used a forward query from the delayed message's timestamp
+      // (previously this would have been a backward query without 'start')
+      const call = (mockClient.chat.queryMAM as ReturnType<typeof vi.fn>).mock.calls[0][0]
+      expect(new Date(call.start).getTime()).toBe(delayedTimestamp.getTime() + 1)
+
+      loadSpy.mockRestore()
+    })
+  })
+
   describe('SM resumption', () => {
     it('should NOT trigger MAM catchup on SM resumption for active conversation', async () => {
       connectionStore.getState().setServerInfo({

packages/fluux-sdk/src/core/chatSideEffects.ts

@@ -18,12 +18,19 @@ import { logInfo } from './logger'
 import { getDomain } from './jid'
 
 /**
- * Find the newest message that was NOT delivered with delay (offline).
- * Delayed messages should not advance the MAM catch-up cursor.
+ * Find the newest message in the array (regardless of delay status).
+ *
+ * Used as the catch-up cursor for MAM forward queries. Previously this
+ * skipped delayed messages, but that caused a subtle bug: when ALL cached
+ * messages are delayed (common after previous MAM catch-ups), the cursor
+ * returned undefined, triggering a backward query. The backward query's
+ * merge (`prependOlderMessages`) incorrectly placed newer messages (sent
+ * from another client while offline) at the top of the list, making them
+ * invisible to the user and preventing the sidebar lastMessage update.
  */
-function findNewestLiveMessage(messages: Array<{ isDelayed?: boolean; timestamp?: Date }>): { timestamp: Date } | undefined {
+function findNewestMessage(messages: Array<{ timestamp?: Date }>): { timestamp: Date } | undefined {
   for (let i = messages.length - 1; i >= 0; i--) {
-    if (!messages[i].isDelayed && messages[i].timestamp) return messages[i] as { timestamp: Date }
+    if (messages[i].timestamp) return messages[i] as { timestamp: Date }
   }
   return undefined
 }
@@ -102,14 +109,11 @@ export function setupChatSideEffects(
 
       // Re-read messages after cache load (store was mutated)
       const cachedMessages = chatStore.getState().messages.get(conversationId) || []
-      // Use the newest non-delayed (live) message as the catch-up cursor.
-      // Delayed messages (offline delivery) should not advance the cursor,
-      // otherwise MAM catch-up skips the gap where originals of corrections live.
-      const newestLiveMessage = findNewestLiveMessage(cachedMessages)
+      const newestMessage = findNewestMessage(cachedMessages)
 
       const queryOptions: { with: string; start?: string } = { with: conversation.id }
-      if (newestLiveMessage?.timestamp) {
-        const startTime = new Date(newestLiveMessage.timestamp.getTime() + 1)
+      if (newestMessage?.timestamp) {
+        const startTime = new Date(newestMessage.timestamp.getTime() + 1)
         queryOptions.start = startTime.toISOString()
       }
 

packages/fluux-sdk/src/core/modules/MAM.catchup.test.ts

@@ -388,7 +388,7 @@ describe('MAM Background Catch-Up', () => {
       expect(mamQuery).toBeDefined()
     })
 
-    it('should fall back to backward query when all cached messages are delayed', async () => {
+    it('should use forward query from newest delayed message as catch-up cursor', async () => {
       await connectClient()
 
       vi.mocked(mockStores.chat.getAllConversations).mockReturnValue([
@@ -415,10 +415,12 @@ describe('MAM Background Catch-Up', () => {
       await waitForAsyncOps(20, 100)
       await catchUpPromise
 
-      // Should fall back to backward query (before="") since no live messages exist
+      // Should use forward query from the newest delayed message timestamp
+      // (not fall back to backward query) so merge uses full sort and
+      // correctly positions messages sent from other clients while offline
       expect(emitSDKSpy).toHaveBeenCalledWith('chat:mam-messages', expect.objectContaining({
         conversationId: 'alice@example.com',
-        direction: 'backward',
+        direction: 'forward',
       }))
     })
   })

packages/fluux-sdk/src/core/modules/MAM.ts

@@ -87,13 +87,17 @@ interface UnresolvedModifications {
 }
 
 /**
- * Find the newest message in an array that was NOT delivered with delay (offline).
- * Delayed messages should not advance the MAM catch-up cursor, otherwise the
- * forward query skips the gap where originals of corrections may live.
+ * Find the newest message in an array (regardless of delay status).
+ *
+ * Used as the catch-up cursor for MAM forward queries. Including delayed
+ * messages ensures the catch-up always uses a forward query, which merges
+ * correctly via full sort. Previously skipping delayed messages caused
+ * backward queries whose prepend-based merge put newer messages (sent
+ * from another client while offline) at the wrong position.
  */
-function findNewestLiveMessage(messages: Array<{ isDelayed?: boolean; timestamp?: Date }>): { timestamp: Date } | undefined {
+function findNewestMessage(messages: Array<{ timestamp?: Date }>): { timestamp: Date } | undefined {
   for (let i = messages.length - 1; i >= 0; i--) {
-    if (!messages[i].isDelayed && messages[i].timestamp) return messages[i] as { timestamp: Date }
+    if (messages[i].timestamp) return messages[i] as { timestamp: Date }
   }
   return undefined
 }
@@ -819,21 +823,18 @@ export class MAM extends BaseModule {
           if (this.deps.stores?.connection.getStatus() !== 'online') return
 
           const messages = conv.messages || []
-          // Use the newest non-delayed (live) message as the catch-up cursor.
-          // Delayed messages (offline delivery) should not advance the cursor,
-          // otherwise MAM catch-up skips the gap where originals of corrections live.
-          const newestLiveMessage = findNewestLiveMessage(messages)
-
-          if (newestLiveMessage?.timestamp) {
-            // Forward query from the last live message
-            const startTime = new Date(newestLiveMessage.timestamp.getTime() + 1)
+          const newestMessage = findNewestMessage(messages)
+
+          if (newestMessage?.timestamp) {
+            // Forward query from the last known message
+            const startTime = new Date(newestMessage.timestamp.getTime() + 1)
             await this.queryArchive({
               with: conv.id,
               start: startTime.toISOString(),
               max: 100,
             })
           } else {
-            // No live messages (all delayed or empty) — fetch latest from MAM
+            // No messages (empty) — fetch latest from MAM
             await this.queryArchive({
               with: conv.id,
               before: '',
@@ -947,19 +948,18 @@ export class MAM extends BaseModule {
           // Re-read room after cache load (store was mutated)
           const updatedRoom = this.deps.stores?.room.getRoom(room.jid)
           const messages = updatedRoom?.messages || []
-          // Use the newest non-delayed (live) message as the catch-up cursor.
-          const newestLiveMessage = findNewestLiveMessage(messages)
+          const newestMessage = findNewestMessage(messages)
 
-          if (newestLiveMessage?.timestamp) {
-            // Forward query from the last live message
-            const startTime = new Date(newestLiveMessage.timestamp.getTime() + 1)
+          if (newestMessage?.timestamp) {
+            // Forward query from the last known message
+            const startTime = new Date(newestMessage.timestamp.getTime() + 1)
             await this.queryRoomArchive({
               roomJid: room.jid,
               start: startTime.toISOString(),
               max: 100,
             })
           } else {
-            // No live messages (all delayed or empty) — fetch latest from MAM
+            // No messages (empty) — fetch latest from MAM
             await this.queryRoomArchive({
               roomJid: room.jid,
               before: '',

packages/fluux-sdk/src/core/roomSideEffects.ts

@@ -20,12 +20,15 @@ import { roomStore, connectionStore } from '../stores'
 import { logInfo } from './logger'
 
 /**
- * Find the newest message that was NOT delivered with delay (offline).
- * Delayed messages should not advance the MAM catch-up cursor.
+ * Find the newest message in the array (regardless of delay status).
+ *
+ * Used as the catch-up cursor for MAM forward queries. Including delayed
+ * messages ensures the catch-up always uses a forward query, which merges
+ * correctly via full sort.
  */
-function findNewestLiveMessage(messages: Array<{ isDelayed?: boolean; timestamp?: Date }>): { timestamp: Date } | undefined {
+function findNewestMessage(messages: Array<{ timestamp?: Date }>): { timestamp: Date } | undefined {
   for (let i = messages.length - 1; i >= 0; i--) {
-    if (!messages[i].isDelayed && messages[i].timestamp) return messages[i] as { timestamp: Date }
+    if (messages[i].timestamp) return messages[i] as { timestamp: Date }
   }
   return undefined
 }
@@ -113,12 +116,11 @@ export function setupRoomSideEffects(
       // Re-read room after cache load (store was mutated)
       const roomAfterCache = roomStore.getState().rooms.get(roomJid)
       const messages = roomAfterCache?.messages || []
-      // Use the newest non-delayed (live) message as the catch-up cursor.
-      const newestLiveMessage = findNewestLiveMessage(messages)
+      const newestMessage = findNewestMessage(messages)
 
-      if (newestLiveMessage?.timestamp) {
-        // Query for messages AFTER the newest live message (catchup)
-        const startTime = new Date(newestLiveMessage.timestamp.getTime() + 1)
+      if (newestMessage?.timestamp) {
+        // Query for messages AFTER the newest known message (catchup)
+        const startTime = new Date(newestMessage.timestamp.getTime() + 1)
         await client.chat.queryRoomMAM({
           roomJid,
           start: startTime.toISOString(),

packages/fluux-sdk/src/stores/shared/messageArrayUtils.test.ts

@@ -687,5 +687,69 @@ describe('messageArrayUtils', () => {
       expect(merged[2].id).toBe('msg-4')
       expect(merged[3].id).toBe('msg-5')
     })
+
+    it('should place newer messages at wrong position (known limitation for backward queries)', () => {
+      // This test documents a known limitation of prependOlderMessages:
+      // If "older" messages are actually NEWER than existing ones, they end up
+      // at the wrong position (prepended before existing instead of appended after).
+      //
+      // This is why catch-up queries must use forward direction (with 'start' filter)
+      // when cached messages exist — mergeAndProcessMessages does a full sort.
+      const existing = [
+        createMessage('msg-1', 'Old cached', new Date('2024-01-15T10:00:00Z')),
+        createMessage('msg-2', 'Old cached', new Date('2024-01-15T11:00:00Z')),
+      ]
+
+      // "Older" messages that are actually newer (e.g., sent from another client while offline)
+      const newer = [
+        createMessage('msg-3', 'Sent from Gajim', new Date('2024-01-15T12:00:00Z')),
+      ]
+
+      const { merged } = prependOlderMessages(existing, newer, (m) => [m.id])
+
+      // BUG: msg-3 (newer) is placed BEFORE msg-1 and msg-2 (older)
+      // This is why the catch-up cursor fix is essential — it prevents this scenario
+      // by always using forward queries (mergeAndProcessMessages) when cached msgs exist
+      expect(merged[0].id).toBe('msg-3') // Wrong: should be at end
+      expect(merged[1].id).toBe('msg-1')
+      expect(merged[2].id).toBe('msg-2')
+    })
+  })
+
+  describe('mergeAndProcessMessages correctly handles catch-up scenario', () => {
+    it('should sort messages sent from another client while offline into correct position', () => {
+      // Simulates the catch-up scenario with forward direction (mergeAndProcessMessages):
+      // - Existing messages loaded from IndexedDB cache (all delayed from previous MAM)
+      // - MAM catch-up returns messages sent from another client while Fluux was offline
+      const existing = [
+        createMessage('msg-1', 'Previous session', new Date('2024-01-15T10:00:00Z')),
+        createMessage('msg-2', 'Previous session', new Date('2024-01-15T11:00:00Z')),
+      ]
+
+      const fromMAM = [
+        // Message sent by user from another client while Fluux was closed
+        createMessage('msg-3', '[OMEMO encrypted]', new Date('2024-01-15T12:00:00Z')),
+        // Reply received while Fluux was closed
+        createMessage('msg-4', '[OMEMO encrypted]', new Date('2024-01-15T13:00:00Z')),
+      ]
+
+      const { merged, newMessages } = mergeAndProcessMessages(
+        existing,
+        fromMAM,
+        (m) => [m.id]
+      )
+
+      // Full sort ensures correct chronological order
+      expect(merged).toHaveLength(4)
+      expect(merged[0].id).toBe('msg-1')
+      expect(merged[1].id).toBe('msg-2')
+      expect(merged[2].id).toBe('msg-3') // Correctly at position 3 (not prepended)
+      expect(merged[3].id).toBe('msg-4') // Correctly at position 4 (newest)
+      expect(newMessages).toHaveLength(2)
+
+      // The last message in the array should be the newest — this is used for
+      // lastMessage sidebar preview in mergeMAMMessages
+      expect(merged[merged.length - 1].id).toBe('msg-4')
+    })
   })
 })