Correctly persist message download state

Add the flag X_DOWNLOADED_FULL when the message has been downloaded completely, the flag X_DOWNLOADED_PARTIAL when only the text of a message has been downloaded, and set no additional flag when only the envelope and structure of the message has been downloaded. The latter happens when we fetch remote search results.

app/core/src/main/java/com/fsck/k9/controller/DraftOperations.kt

@@ -7,6 +7,7 @@ import com.fsck.k9.controller.MessagingControllerCommands.PendingAppend
 import com.fsck.k9.controller.MessagingControllerCommands.PendingReplace
 import com.fsck.k9.mail.FetchProfile
 import com.fsck.k9.mail.Message
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.mail.MessagingException
 import com.fsck.k9.mailstore.LocalFolder
 import com.fsck.k9.mailstore.LocalMessage
@@ -164,6 +165,6 @@ internal class DraftOperations(
     }
 
     private fun Message.toSaveMessageData(subject: String?): SaveMessageData {
-        return saveMessageDataCreator.createSaveMessageData(this, partialMessage = false, subject)
+        return saveMessageDataCreator.createSaveMessageData(this, MessageDownloadState.FULL, subject)
     }
 }

app/core/src/main/java/com/fsck/k9/controller/MessagingController.java

@@ -67,6 +67,7 @@ import com.fsck.k9.mailstore.LocalFolder;
 import com.fsck.k9.mailstore.LocalMessage;
 import com.fsck.k9.mailstore.LocalStore;
 import com.fsck.k9.mailstore.LocalStoreProvider;
+import com.fsck.k9.mail.MessageDownloadState;
 import com.fsck.k9.mailstore.MessageStore;
 import com.fsck.k9.mailstore.MessageStoreManager;
 import com.fsck.k9.mailstore.OutboxState;
@@ -1375,7 +1376,8 @@ public class MessagingController {
             message.setFlag(Flag.SEEN, true);
 
             MessageStore messageStore = messageStoreManager.getMessageStore(account);
-            SaveMessageData messageData = saveMessageDataCreator.createSaveMessageData(message, false, plaintextSubject);
+            SaveMessageData messageData = saveMessageDataCreator.createSaveMessageData(
+                    message, MessageDownloadState.FULL, plaintextSubject);
             long messageId = messageStore.saveLocalMessage(outboxFolderId, messageData, null);
 
             LocalStore localStore = localStoreProvider.getInstance(account);

app/core/src/main/java/com/fsck/k9/mailstore/K9BackendFolder.kt

@@ -4,6 +4,7 @@ import com.fsck.k9.backend.api.BackendFolder
 import com.fsck.k9.backend.api.BackendFolder.MoreMessages
 import com.fsck.k9.mail.Flag
 import com.fsck.k9.mail.Message
+import com.fsck.k9.mail.MessageDownloadState
 import java.util.Date
 import com.fsck.k9.mailstore.MoreMessages as StoreMoreMessages
 
@@ -85,18 +86,10 @@ class K9BackendFolder(
         messageStore.setMessageFlag(folderId, messageServerId, flag, value)
     }
 
-    override fun saveCompleteMessage(message: Message) {
+    override fun saveMessage(message: Message, downloadState: MessageDownloadState) {
-        saveMessage(message, partialMessage = false)
-    }
-
-    override fun savePartialMessage(message: Message) {
-        saveMessage(message, partialMessage = true)
-    }
-
-    private fun saveMessage(message: Message, partialMessage: Boolean) {
         requireMessageServerId(message)
 
-        val messageData = saveMessageDataCreator.createSaveMessageData(message, partialMessage)
+        val messageData = saveMessageDataCreator.createSaveMessageData(message, downloadState)
         messageStore.saveRemoteMessage(folderId, message.uid, messageData)
     }
 

app/core/src/main/java/com/fsck/k9/mailstore/SaveMessageData.kt

@@ -1,6 +1,7 @@
 package com.fsck.k9.mailstore
 
 import com.fsck.k9.mail.Message
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.message.extractors.PreviewResult
 
 data class SaveMessageData(
@@ -8,7 +9,7 @@ data class SaveMessageData(
     val subject: String?,
     val date: Long,
     val internalDate: Long,
-    val partialMessage: Boolean,
+    val downloadState: MessageDownloadState,
     val attachmentCount: Int,
     val previewResult: PreviewResult,
     val textForSearchIndex: String? = null,

app/core/src/main/java/com/fsck/k9/mailstore/SaveMessageDataCreator.kt

@@ -2,6 +2,7 @@ package com.fsck.k9.mailstore
 
 import com.fsck.k9.crypto.EncryptionExtractor
 import com.fsck.k9.mail.Message
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.message.extractors.AttachmentCounter
 import com.fsck.k9.message.extractors.MessageFulltextCreator
 import com.fsck.k9.message.extractors.MessagePreviewCreator
@@ -12,7 +13,11 @@ class SaveMessageDataCreator(
     private val messageFulltextCreator: MessageFulltextCreator,
     private val attachmentCounter: AttachmentCounter
 ) {
-    fun createSaveMessageData(message: Message, partialMessage: Boolean, subject: String? = null): SaveMessageData {
+    fun createSaveMessageData(
+        message: Message,
+        downloadState: MessageDownloadState,
+        subject: String? = null
+    ): SaveMessageData {
         val now = System.currentTimeMillis()
         val date = message.sentDate?.time ?: now
         val internalDate = message.internalDate?.time ?: now
@@ -25,7 +30,7 @@ class SaveMessageDataCreator(
                 subject = displaySubject,
                 date = date,
                 internalDate = internalDate,
-                partialMessage = partialMessage,
+                downloadState = downloadState,
                 attachmentCount = encryptionResult.attachmentCount,
                 previewResult = encryptionResult.previewResult,
                 textForSearchIndex = encryptionResult.textForSearchIndex,
@@ -37,7 +42,7 @@ class SaveMessageDataCreator(
                 subject = displaySubject,
                 date = date,
                 internalDate = internalDate,
-                partialMessage = partialMessage,
+                downloadState = downloadState,
                 attachmentCount = attachmentCounter.getAttachmentCount(message),
                 previewResult = messagePreviewCreator.createPreview(message),
                 textForSearchIndex = messageFulltextCreator.createFulltext(message),

app/core/src/test/java/com/fsck/k9/mailstore/K9BackendFolderTest.kt

@@ -13,6 +13,7 @@ import com.fsck.k9.mail.Address
 import com.fsck.k9.mail.Flag
 import com.fsck.k9.mail.FolderType
 import com.fsck.k9.mail.Message
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.mail.internet.MimeMessage
 import com.fsck.k9.mail.internet.MimeMessageHelper
 import com.fsck.k9.mail.internet.TextBody
@@ -92,7 +93,7 @@ class K9BackendFolderTest : K9RobolectricTest() {
         val message = createMessage(messageServerId = null)
 
         try {
-            backendFolder.saveCompleteMessage(message)
+            backendFolder.saveMessage(message, MessageDownloadState.FULL)
             fail("Expected exception")
         } catch (e: IllegalStateException) {
         }
@@ -103,7 +104,7 @@ class K9BackendFolderTest : K9RobolectricTest() {
         val message = createMessage(messageServerId = null)
 
         try {
-            backendFolder.savePartialMessage(message)
+            backendFolder.saveMessage(message, MessageDownloadState.PARTIAL)
             fail("Expected exception")
         } catch (e: IllegalStateException) {
         }
@@ -136,7 +137,7 @@ class K9BackendFolderTest : K9RobolectricTest() {
 
     fun createMessageInBackendFolder(messageServerId: String, flags: Set<Flag> = emptySet()) {
         val message = createMessage(messageServerId, flags)
-        backendFolder.saveCompleteMessage(message)
+        backendFolder.saveMessage(message, MessageDownloadState.FULL)
 
         val messageServerIds = backendFolder.getMessageServerIds()
         assertTrue(messageServerId in messageServerIds)

app/storage/src/main/java/com/fsck/k9/storage/messages/SaveMessageOperations.kt

@@ -9,6 +9,7 @@ import com.fsck.k9.mail.BoundaryGenerator
 import com.fsck.k9.mail.Flag
 import com.fsck.k9.mail.Message
 import com.fsck.k9.mail.Message.RecipientType
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.mail.Multipart
 import com.fsck.k9.mail.Part
 import com.fsck.k9.mail.filter.CountingOutputStream
@@ -381,10 +382,10 @@ internal class SaveMessageOperations(
     ): Long {
         val message = messageData.message
 
-        if (messageData.partialMessage) {
+        when (messageData.downloadState) {
-            message.setFlag(Flag.X_DOWNLOADED_PARTIAL, true)
+            MessageDownloadState.ENVELOPE -> Unit
-        } else {
+            MessageDownloadState.PARTIAL -> message.setFlag(Flag.X_DOWNLOADED_PARTIAL, true)
-            message.setFlag(Flag.X_DOWNLOADED_FULL, true)
+            MessageDownloadState.FULL -> message.setFlag(Flag.X_DOWNLOADED_FULL, true)
         }
 
         val values = ContentValues().apply {

app/storage/src/test/java/com/fsck/k9/storage/messages/SaveMessageOperationsTest.kt

@@ -4,6 +4,7 @@ import com.fsck.k9.K9
 import com.fsck.k9.mail.Address
 import com.fsck.k9.mail.Flag
 import com.fsck.k9.mail.Message
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.mail.Multipart
 import com.fsck.k9.mail.Part
 import com.fsck.k9.mail.buildMessage
@@ -500,7 +501,7 @@ class SaveMessageOperationsTest : RobolectricTest() {
         subject: String? = getSubject(),
         date: Long = sentDate?.time ?: System.currentTimeMillis(),
         internalDate: Long = date,
-        partialMessage: Boolean = isPartialMessage(),
+        downloadState: MessageDownloadState = getDownloadState(),
         attachmentCount: Int = 0,
         previewResult: PreviewResult = PreviewResult.none(),
         textForSearchIndex: String? = null,
@@ -511,7 +512,7 @@ class SaveMessageOperationsTest : RobolectricTest() {
             subject,
             date,
             internalDate,
-            partialMessage,
+            downloadState,
             attachmentCount,
             previewResult,
             textForSearchIndex,
@@ -519,14 +520,16 @@ class SaveMessageOperationsTest : RobolectricTest() {
         )
     }
 
-    private fun Message.isPartialMessage(): Boolean {
+    private fun Message.getDownloadState(): MessageDownloadState {
+        if (body == null) return MessageDownloadState.ENVELOPE
+
         val stack = Stack<Part>()
         stack.push(this)
 
         while (stack.isNotEmpty()) {
             val part = stack.pop()
             when (val body = part.body) {
-                null -> return true
+                null -> return MessageDownloadState.PARTIAL
                 is Multipart -> {
                     for (i in 0 until body.count) {
                         stack.push(body.getBodyPart(i))
@@ -535,7 +538,7 @@ class SaveMessageOperationsTest : RobolectricTest() {
             }
         }
 
-        return false
+        return MessageDownloadState.FULL
     }
 
     private fun Message.header(): String {

backend/api/src/main/java/com/fsck/k9/backend/api/BackendFolder.kt

@@ -2,6 +2,7 @@ package com.fsck.k9.backend.api
 
 import com.fsck.k9.mail.Flag
 import com.fsck.k9.mail.Message
+import com.fsck.k9.mail.MessageDownloadState
 import java.util.Date
 
 // FIXME: add documentation
@@ -21,8 +22,7 @@ interface BackendFolder {
     fun isMessagePresent(messageServerId: String): Boolean
     fun getMessageFlags(messageServerId: String): Set<Flag>
     fun setMessageFlag(messageServerId: String, flag: Flag, value: Boolean)
-    fun savePartialMessage(message: Message)
+    fun saveMessage(message: Message, downloadState: MessageDownloadState)
-    fun saveCompleteMessage(message: Message)
     fun getOldestMessageDate(): Date?
     fun getFolderExtraString(name: String): String?
     fun setFolderExtraString(name: String, value: String?)

backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandDownloadMessage.kt

@@ -6,6 +6,7 @@ import com.fsck.k9.mail.FetchProfile.Item.BODY
 import com.fsck.k9.mail.FetchProfile.Item.ENVELOPE
 import com.fsck.k9.mail.FetchProfile.Item.FLAGS
 import com.fsck.k9.mail.FetchProfile.Item.STRUCTURE
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.mail.helper.fetchProfileOf
 import com.fsck.k9.mail.store.imap.ImapFolder
 import com.fsck.k9.mail.store.imap.ImapMessage
@@ -25,7 +26,7 @@ internal class CommandDownloadMessage(private val backendStorage: BackendStorage
             fetchMessage(folder, message, fetchProfileOf(STRUCTURE))
 
             val backendFolder = backendStorage.getFolder(folderServerId)
-            backendFolder.savePartialMessage(message)
+            backendFolder.saveMessage(message, MessageDownloadState.ENVELOPE)
         } finally {
             folder.close()
         }
@@ -40,7 +41,7 @@ internal class CommandDownloadMessage(private val backendStorage: BackendStorage
             fetchMessage(folder, message, fetchProfileOf(FLAGS, BODY))
 
             val backendFolder = backendStorage.getFolder(folderServerId)
-            backendFolder.saveCompleteMessage(message)
+            backendFolder.saveMessage(message, MessageDownloadState.FULL)
         } finally {
             folder.close()
         }

backend/imap/src/main/java/com/fsck/k9/backend/imap/ImapSync.kt

@@ -12,6 +12,7 @@ import com.fsck.k9.mail.BodyFactory
 import com.fsck.k9.mail.DefaultBodyFactory
 import com.fsck.k9.mail.FetchProfile
 import com.fsck.k9.mail.Flag
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.mail.MessageRetrievalListener
 import com.fsck.k9.mail.internet.MessageExtractor
 import com.fsck.k9.mail.store.imap.ImapFolder
@@ -526,7 +527,7 @@ internal class ImapSync(
                 override fun messageFinished(message: ImapMessage, number: Int, ofTotal: Int) {
                     try {
                         // Store the updated message locally
-                        backendFolder.saveCompleteMessage(message)
+                        backendFolder.saveMessage(message, MessageDownloadState.FULL)
                         progress.incrementAndGet()
 
                         // Increment the number of "new messages" if the newly downloaded message is not marked as read.
@@ -664,7 +665,7 @@ internal class ImapSync(
         remoteFolder.fetch(listOf(message), fetchProfile, null, maxDownloadSize)
 
         // Store the updated message locally
-        backendFolder.savePartialMessage(message)
+        backendFolder.saveMessage(message, MessageDownloadState.PARTIAL)
     }
 
     private fun downloadPartial(
@@ -690,7 +691,7 @@ internal class ImapSync(
         }
 
         // Store the updated message locally
-        backendFolder.savePartialMessage(message)
+        backendFolder.saveMessage(message, MessageDownloadState.PARTIAL)
     }
 
     private fun syncFlags(syncConfig: SyncConfig, backendFolder: BackendFolder, remoteMessage: ImapMessage): Boolean {

backend/jmap/src/main/java/com/fsck/k9/backend/jmap/CommandSync.kt

@@ -6,6 +6,7 @@ import com.fsck.k9.backend.api.SyncConfig
 import com.fsck.k9.backend.api.SyncListener
 import com.fsck.k9.mail.AuthenticationFailedException
 import com.fsck.k9.mail.Flag
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.mail.internet.MimeMessage
 import java.util.Date
 import okhttp3.HttpUrl
@@ -192,7 +193,7 @@ class CommandSync(
                     setFlags(messageInfo.flags, true)
                 }
 
-                backendFolder.saveCompleteMessage(message)
+                backendFolder.saveMessage(message, MessageDownloadState.FULL)
             } else {
                 Timber.d("Failed to download message: %s", messageInfo.serverId)
             }

backend/jmap/src/test/java/com/fsck/k9/backend/jmap/InMemoryBackendFolder.kt

@@ -4,6 +4,7 @@ import com.fsck.k9.backend.api.BackendFolder
 import com.fsck.k9.mail.Flag
 import com.fsck.k9.mail.FolderType
 import com.fsck.k9.mail.Message
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.mail.internet.MimeMessage
 import java.util.Date
 import okio.Buffer
@@ -108,16 +109,18 @@ class InMemoryBackendFolder(override var name: String, var type: FolderType) : B
         }
     }
 
-    override fun savePartialMessage(message: Message) {
+    override fun saveMessage(message: Message, downloadState: MessageDownloadState) {
         val messageServerId = checkNotNull(message.uid)
         messages[messageServerId] = message
-        messageFlags[messageServerId] = message.flags.toMutableSet()
+        val flags = message.flags.toMutableSet()
+
+        when (downloadState) {
+            MessageDownloadState.ENVELOPE -> Unit
+            MessageDownloadState.PARTIAL -> flags.add(Flag.X_DOWNLOADED_PARTIAL)
+            MessageDownloadState.FULL -> flags.add(Flag.X_DOWNLOADED_FULL)
         }
 
-    override fun saveCompleteMessage(message: Message) {
+        messageFlags[messageServerId] = flags
-        val messageServerId = checkNotNull(message.uid)
-        messages[messageServerId] = message
-        messageFlags[messageServerId] = message.flags.toMutableSet()
     }
 
     override fun getOldestMessageDate(): Date? {

backend/pop3/src/main/java/com/fsck/k9/backend/pop3/CommandDownloadMessage.kt

@@ -3,6 +3,7 @@ package com.fsck.k9.backend.pop3
 import com.fsck.k9.backend.api.BackendStorage
 import com.fsck.k9.mail.FetchProfile.Item.BODY
 import com.fsck.k9.mail.FetchProfile.Item.FLAGS
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.mail.helper.fetchProfileOf
 import com.fsck.k9.mail.store.pop3.Pop3Store
 
@@ -17,7 +18,7 @@ internal class CommandDownloadMessage(private val backendStorage: BackendStorage
             folder.fetch(listOf(message), fetchProfileOf(FLAGS, BODY), null, 0)
 
             val backendFolder = backendStorage.getFolder(folderServerId)
-            backendFolder.saveCompleteMessage(message)
+            backendFolder.saveMessage(message, MessageDownloadState.FULL)
         } finally {
             folder.close()
         }

backend/pop3/src/main/java/com/fsck/k9/backend/pop3/Pop3Sync.java

@@ -10,6 +10,7 @@ import com.fsck.k9.helper.ExceptionHelper;
 import com.fsck.k9.mail.AuthenticationFailedException;
 import com.fsck.k9.mail.FetchProfile;
 import com.fsck.k9.mail.Flag;
+import com.fsck.k9.mail.MessageDownloadState;
 import com.fsck.k9.mail.MessageRetrievalListener;
 import com.fsck.k9.mail.MessagingException;
 import com.fsck.k9.mail.store.pop3.Pop3Folder;
@@ -377,9 +378,9 @@ class Pop3Sync {
                 // Store the updated message locally
                 boolean completeMessage = message.isSet(Flag.X_DOWNLOADED_FULL);
                 if (completeMessage) {
-                    backendFolder.saveCompleteMessage(message);
+                    backendFolder.saveMessage(message, MessageDownloadState.FULL);
                 } else {
-                    backendFolder.savePartialMessage(message);
+                    backendFolder.saveMessage(message, MessageDownloadState.PARTIAL);
                 }
 
                 boolean isOldMessage = isOldMessage(backendFolder, message);
@@ -480,7 +481,7 @@ class Pop3Sync {
                         try {
 
                             // Store the updated message locally
-                            backendFolder.saveCompleteMessage(message);
+                            backendFolder.saveMessage(message, MessageDownloadState.FULL);
                             progress.incrementAndGet();
 
                             // Increment the number of "new messages" if the newly downloaded message is
@@ -603,9 +604,9 @@ class Pop3Sync {
 
         // Store the updated message locally
         if (completeMessage) {
-            backendFolder.saveCompleteMessage(message);
+            backendFolder.saveMessage(message, MessageDownloadState.FULL);
         } else {
-            backendFolder.savePartialMessage(message);
+            backendFolder.saveMessage(message, MessageDownloadState.PARTIAL);
         }
     }
 }

backend/webdav/src/main/java/com/fsck/k9/backend/webdav/CommandDownloadMessage.kt

@@ -3,6 +3,7 @@ package com.fsck.k9.backend.webdav
 import com.fsck.k9.backend.api.BackendStorage
 import com.fsck.k9.mail.FetchProfile.Item.BODY
 import com.fsck.k9.mail.FetchProfile.Item.FLAGS
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.mail.helper.fetchProfileOf
 import com.fsck.k9.mail.store.webdav.WebDavStore
 
@@ -16,7 +17,7 @@ internal class CommandDownloadMessage(val backendStorage: BackendStorage, privat
             folder.fetch(listOf(message), fetchProfileOf(FLAGS, BODY), null, 0)
 
             val backendFolder = backendStorage.getFolder(folderServerId)
-            backendFolder.saveCompleteMessage(message)
+            backendFolder.saveMessage(message, MessageDownloadState.FULL)
         } finally {
             folder.close()
         }

backend/webdav/src/main/java/com/fsck/k9/backend/webdav/WebDavSync.java

@@ -10,6 +10,7 @@ import com.fsck.k9.helper.ExceptionHelper;
 import com.fsck.k9.mail.AuthenticationFailedException;
 import com.fsck.k9.mail.FetchProfile;
 import com.fsck.k9.mail.Flag;
+import com.fsck.k9.mail.MessageDownloadState;
 import com.fsck.k9.mail.MessageRetrievalListener;
 import com.fsck.k9.mail.MessagingException;
 import com.fsck.k9.mail.store.webdav.WebDavFolder;
@@ -364,9 +365,9 @@ class WebDavSync {
                 // Store the updated message locally
                 boolean completeMessage = message.isSet(Flag.X_DOWNLOADED_FULL);
                 if (completeMessage) {
-                    backendFolder.saveCompleteMessage(message);
+                    backendFolder.saveMessage(message, MessageDownloadState.FULL);
                 } else {
-                    backendFolder.savePartialMessage(message);
+                    backendFolder.saveMessage(message, MessageDownloadState.PARTIAL);
                 }
 
                 listener.syncNewMessage(folder, messageServerId, false);
@@ -466,7 +467,7 @@ class WebDavSync {
                         try {
 
                             // Store the updated message locally
-                            backendFolder.saveCompleteMessage(message);
+                            backendFolder.saveMessage(message, MessageDownloadState.FULL);
                             progress.incrementAndGet();
 
                             // Increment the number of "new messages" if the newly downloaded message is
@@ -582,9 +583,9 @@ class WebDavSync {
 
         // Store the updated message locally
         if (completeMessage) {
-            backendFolder.saveCompleteMessage(message);
+            backendFolder.saveMessage(message, MessageDownloadState.FULL);
         } else {
-            backendFolder.savePartialMessage(message);
+            backendFolder.saveMessage(message, MessageDownloadState.PARTIAL);
         }
     }
 

mail/common/src/main/java/com/fsck/k9/mail/MessageDownloadState.kt

@@ -0,0 +1,7 @@
+package com.fsck.k9.mail
+
+enum class MessageDownloadState {
+    ENVELOPE,
+    PARTIAL,
+    FULL
+}