Merge pull request #5293 from k9mail/download_state

Correctly persist message download state

app/core/src/main/java/com/fsck/k9/controller/DraftOperations.kt

@@ -7,6 +7,7 @@ import com.fsck.k9.controller.MessagingControllerCommands.PendingAppend
 import com.fsck.k9.controller.MessagingControllerCommands.PendingReplace
 import com.fsck.k9.mail.FetchProfile
 import com.fsck.k9.mail.Message
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.mail.MessagingException
 import com.fsck.k9.mailstore.LocalFolder
 import com.fsck.k9.mailstore.LocalMessage
@@ -164,6 +165,6 @@ internal class DraftOperations(
     }
 
     private fun Message.toSaveMessageData(subject: String?): SaveMessageData {
-        return saveMessageDataCreator.createSaveMessageData(this, partialMessage = false, subject)
+        return saveMessageDataCreator.createSaveMessageData(this, MessageDownloadState.FULL, subject)
     }
 }

app/core/src/main/java/com/fsck/k9/controller/MessagingController.java

@@ -67,6 +67,7 @@ import com.fsck.k9.mailstore.LocalFolder;
 import com.fsck.k9.mailstore.LocalMessage;
 import com.fsck.k9.mailstore.LocalStore;
 import com.fsck.k9.mailstore.LocalStoreProvider;
+import com.fsck.k9.mail.MessageDownloadState;
 import com.fsck.k9.mailstore.MessageStore;
 import com.fsck.k9.mailstore.MessageStoreManager;
 import com.fsck.k9.mailstore.OutboxState;
@@ -1375,7 +1376,8 @@ public class MessagingController {
             message.setFlag(Flag.SEEN, true);
 
             MessageStore messageStore = messageStoreManager.getMessageStore(account);
-            SaveMessageData messageData = saveMessageDataCreator.createSaveMessageData(message, false, plaintextSubject);
+            SaveMessageData messageData = saveMessageDataCreator.createSaveMessageData(
+                    message, MessageDownloadState.FULL, plaintextSubject);
             long messageId = messageStore.saveLocalMessage(outboxFolderId, messageData, null);
 
             LocalStore localStore = localStoreProvider.getInstance(account);

app/core/src/main/java/com/fsck/k9/mailstore/K9BackendFolder.kt

@@ -4,6 +4,7 @@ import com.fsck.k9.backend.api.BackendFolder
 import com.fsck.k9.backend.api.BackendFolder.MoreMessages
 import com.fsck.k9.mail.Flag
 import com.fsck.k9.mail.Message
+import com.fsck.k9.mail.MessageDownloadState
 import java.util.Date
 import com.fsck.k9.mailstore.MoreMessages as StoreMoreMessages
 
@@ -85,18 +86,10 @@ class K9BackendFolder(
         messageStore.setMessageFlag(folderId, messageServerId, flag, value)
     }
 
-    override fun saveCompleteMessage(message: Message) {
+    override fun saveMessage(message: Message, downloadState: MessageDownloadState) {
-        saveMessage(message, partialMessage = false)
-    }
-
-    override fun savePartialMessage(message: Message) {
-        saveMessage(message, partialMessage = true)
-    }
-
-    private fun saveMessage(message: Message, partialMessage: Boolean) {
         requireMessageServerId(message)
 
-        val messageData = saveMessageDataCreator.createSaveMessageData(message, partialMessage)
+        val messageData = saveMessageDataCreator.createSaveMessageData(message, downloadState)
         messageStore.saveRemoteMessage(folderId, message.uid, messageData)
     }
 

app/core/src/main/java/com/fsck/k9/mailstore/SaveMessageData.kt

@@ -1,6 +1,7 @@
 package com.fsck.k9.mailstore
 
 import com.fsck.k9.mail.Message
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.message.extractors.PreviewResult
 
 data class SaveMessageData(
@@ -8,7 +9,7 @@ data class SaveMessageData(
     val subject: String?,
     val date: Long,
     val internalDate: Long,
-    val partialMessage: Boolean,
+    val downloadState: MessageDownloadState,
     val attachmentCount: Int,
     val previewResult: PreviewResult,
     val textForSearchIndex: String? = null,

app/core/src/main/java/com/fsck/k9/mailstore/SaveMessageDataCreator.kt

@@ -2,6 +2,7 @@ package com.fsck.k9.mailstore
 
 import com.fsck.k9.crypto.EncryptionExtractor
 import com.fsck.k9.mail.Message
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.message.extractors.AttachmentCounter
 import com.fsck.k9.message.extractors.MessageFulltextCreator
 import com.fsck.k9.message.extractors.MessagePreviewCreator
@@ -12,7 +13,11 @@ class SaveMessageDataCreator(
     private val messageFulltextCreator: MessageFulltextCreator,
     private val attachmentCounter: AttachmentCounter
 ) {
-    fun createSaveMessageData(message: Message, partialMessage: Boolean, subject: String? = null): SaveMessageData {
+    fun createSaveMessageData(
+        message: Message,
+        downloadState: MessageDownloadState,
+        subject: String? = null
+    ): SaveMessageData {
         val now = System.currentTimeMillis()
         val date = message.sentDate?.time ?: now
         val internalDate = message.internalDate?.time ?: now
@@ -25,7 +30,7 @@ class SaveMessageDataCreator(
                 subject = displaySubject,
                 date = date,
                 internalDate = internalDate,
-                partialMessage = partialMessage,
+                downloadState = downloadState,
                 attachmentCount = encryptionResult.attachmentCount,
                 previewResult = encryptionResult.previewResult,
                 textForSearchIndex = encryptionResult.textForSearchIndex,
@@ -37,7 +42,7 @@ class SaveMessageDataCreator(
                 subject = displaySubject,
                 date = date,
                 internalDate = internalDate,
-                partialMessage = partialMessage,
+                downloadState = downloadState,
                 attachmentCount = attachmentCounter.getAttachmentCount(message),
                 previewResult = messagePreviewCreator.createPreview(message),
                 textForSearchIndex = messageFulltextCreator.createFulltext(message),

app/core/src/test/java/com/fsck/k9/mailstore/K9BackendFolderTest.kt

@@ -13,6 +13,7 @@ import com.fsck.k9.mail.Address
 import com.fsck.k9.mail.Flag
 import com.fsck.k9.mail.FolderType
 import com.fsck.k9.mail.Message
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.mail.internet.MimeMessage
 import com.fsck.k9.mail.internet.MimeMessageHelper
 import com.fsck.k9.mail.internet.TextBody
@@ -92,7 +93,7 @@ class K9BackendFolderTest : K9RobolectricTest() {
         val message = createMessage(messageServerId = null)
 
         try {
-            backendFolder.saveCompleteMessage(message)
+            backendFolder.saveMessage(message, MessageDownloadState.FULL)
             fail("Expected exception")
         } catch (e: IllegalStateException) {
         }
@@ -103,7 +104,7 @@ class K9BackendFolderTest : K9RobolectricTest() {
         val message = createMessage(messageServerId = null)
 
         try {
-            backendFolder.savePartialMessage(message)
+            backendFolder.saveMessage(message, MessageDownloadState.PARTIAL)
             fail("Expected exception")
         } catch (e: IllegalStateException) {
         }
@@ -136,7 +137,7 @@ class K9BackendFolderTest : K9RobolectricTest() {
 
     fun createMessageInBackendFolder(messageServerId: String, flags: Set<Flag> = emptySet()) {
         val message = createMessage(messageServerId, flags)
-        backendFolder.saveCompleteMessage(message)
+        backendFolder.saveMessage(message, MessageDownloadState.FULL)
 
         val messageServerIds = backendFolder.getMessageServerIds()
         assertTrue(messageServerId in messageServerIds)

app/storage/src/main/java/com/fsck/k9/storage/messages/SaveMessageOperations.kt

@@ -9,6 +9,7 @@ import com.fsck.k9.mail.BoundaryGenerator
 import com.fsck.k9.mail.Flag
 import com.fsck.k9.mail.Message
 import com.fsck.k9.mail.Message.RecipientType
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.mail.Multipart
 import com.fsck.k9.mail.Part
 import com.fsck.k9.mail.filter.CountingOutputStream
@@ -381,10 +382,10 @@ internal class SaveMessageOperations(
     ): Long {
         val message = messageData.message
 
-        if (messageData.partialMessage) {
+        when (messageData.downloadState) {
-            message.setFlag(Flag.X_DOWNLOADED_PARTIAL, true)
+            MessageDownloadState.ENVELOPE -> Unit
-        } else {
+            MessageDownloadState.PARTIAL -> message.setFlag(Flag.X_DOWNLOADED_PARTIAL, true)
-            message.setFlag(Flag.X_DOWNLOADED_FULL, true)
+            MessageDownloadState.FULL -> message.setFlag(Flag.X_DOWNLOADED_FULL, true)
         }
 
         val values = ContentValues().apply {

app/storage/src/test/java/com/fsck/k9/storage/messages/SaveMessageOperationsTest.kt

@@ -4,6 +4,7 @@ import com.fsck.k9.K9
 import com.fsck.k9.mail.Address
 import com.fsck.k9.mail.Flag
 import com.fsck.k9.mail.Message
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.mail.Multipart
 import com.fsck.k9.mail.Part
 import com.fsck.k9.mail.buildMessage
@@ -500,7 +501,7 @@ class SaveMessageOperationsTest : RobolectricTest() {
         subject: String? = getSubject(),
         date: Long = sentDate?.time ?: System.currentTimeMillis(),
         internalDate: Long = date,
-        partialMessage: Boolean = isPartialMessage(),
+        downloadState: MessageDownloadState = getDownloadState(),
         attachmentCount: Int = 0,
         previewResult: PreviewResult = PreviewResult.none(),
         textForSearchIndex: String? = null,
@@ -511,7 +512,7 @@ class SaveMessageOperationsTest : RobolectricTest() {
             subject,
             date,
             internalDate,
-            partialMessage,
+            downloadState,
             attachmentCount,
             previewResult,
             textForSearchIndex,
@@ -519,14 +520,16 @@ class SaveMessageOperationsTest : RobolectricTest() {
         )
     }
 
-    private fun Message.isPartialMessage(): Boolean {
+    private fun Message.getDownloadState(): MessageDownloadState {
+        if (body == null) return MessageDownloadState.ENVELOPE
+
         val stack = Stack<Part>()
         stack.push(this)
 
         while (stack.isNotEmpty()) {
             val part = stack.pop()
             when (val body = part.body) {
-                null -> return true
+                null -> return MessageDownloadState.PARTIAL
                 is Multipart -> {
                     for (i in 0 until body.count) {
                         stack.push(body.getBodyPart(i))
@@ -535,7 +538,7 @@ class SaveMessageOperationsTest : RobolectricTest() {
             }
         }
 
-        return false
+        return MessageDownloadState.FULL
     }
 
     private fun Message.header(): String {

backend/api/src/main/java/com/fsck/k9/backend/api/BackendFolder.kt

@@ -2,6 +2,7 @@ package com.fsck.k9.backend.api
 
 import com.fsck.k9.mail.Flag
 import com.fsck.k9.mail.Message
+import com.fsck.k9.mail.MessageDownloadState
 import java.util.Date
 
 // FIXME: add documentation
@@ -21,8 +22,7 @@ interface BackendFolder {
     fun isMessagePresent(messageServerId: String): Boolean
     fun getMessageFlags(messageServerId: String): Set<Flag>
     fun setMessageFlag(messageServerId: String, flag: Flag, value: Boolean)
-    fun savePartialMessage(message: Message)
+    fun saveMessage(message: Message, downloadState: MessageDownloadState)
-    fun saveCompleteMessage(message: Message)
     fun getOldestMessageDate(): Date?
     fun getFolderExtraString(name: String): String?
     fun setFolderExtraString(name: String, value: String?)

backend/imap/src/main/java/com/fsck/k9/backend/imap/CommandDownloadMessage.kt

@@ -6,6 +6,7 @@ import com.fsck.k9.mail.FetchProfile.Item.BODY
 import com.fsck.k9.mail.FetchProfile.Item.ENVELOPE
 import com.fsck.k9.mail.FetchProfile.Item.FLAGS
 import com.fsck.k9.mail.FetchProfile.Item.STRUCTURE
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.mail.helper.fetchProfileOf
 import com.fsck.k9.mail.store.imap.ImapFolder
 import com.fsck.k9.mail.store.imap.ImapMessage
@@ -25,7 +26,7 @@ internal class CommandDownloadMessage(private val backendStorage: BackendStorage
             fetchMessage(folder, message, fetchProfileOf(STRUCTURE))
 
             val backendFolder = backendStorage.getFolder(folderServerId)
-            backendFolder.savePartialMessage(message)
+            backendFolder.saveMessage(message, MessageDownloadState.ENVELOPE)
         } finally {
             folder.close()
         }
@@ -40,7 +41,7 @@ internal class CommandDownloadMessage(private val backendStorage: BackendStorage
             fetchMessage(folder, message, fetchProfileOf(FLAGS, BODY))
 
             val backendFolder = backendStorage.getFolder(folderServerId)
-            backendFolder.saveCompleteMessage(message)
+            backendFolder.saveMessage(message, MessageDownloadState.FULL)
         } finally {
             folder.close()
         }

backend/imap/src/main/java/com/fsck/k9/backend/imap/ImapSync.kt

@@ -12,6 +12,7 @@ import com.fsck.k9.mail.BodyFactory
 import com.fsck.k9.mail.DefaultBodyFactory
 import com.fsck.k9.mail.FetchProfile
 import com.fsck.k9.mail.Flag
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.mail.MessageRetrievalListener
 import com.fsck.k9.mail.internet.MessageExtractor
 import com.fsck.k9.mail.store.imap.ImapFolder
@@ -526,7 +527,7 @@ internal class ImapSync(
                 override fun messageFinished(message: ImapMessage, number: Int, ofTotal: Int) {
                     try {
                         // Store the updated message locally
-                        backendFolder.saveCompleteMessage(message)
+                        backendFolder.saveMessage(message, MessageDownloadState.FULL)
                         progress.incrementAndGet()
 
                         // Increment the number of "new messages" if the newly downloaded message is not marked as read.
@@ -664,7 +665,7 @@ internal class ImapSync(
         remoteFolder.fetch(listOf(message), fetchProfile, null, maxDownloadSize)
 
         // Store the updated message locally
-        backendFolder.savePartialMessage(message)
+        backendFolder.saveMessage(message, MessageDownloadState.PARTIAL)
     }
 
     private fun downloadPartial(
@@ -690,7 +691,7 @@ internal class ImapSync(
         }
 
         // Store the updated message locally
-        backendFolder.savePartialMessage(message)
+        backendFolder.saveMessage(message, MessageDownloadState.PARTIAL)
     }
 
     private fun syncFlags(syncConfig: SyncConfig, backendFolder: BackendFolder, remoteMessage: ImapMessage): Boolean {

backend/jmap/src/main/java/com/fsck/k9/backend/jmap/CommandSync.kt

@@ -6,6 +6,7 @@ import com.fsck.k9.backend.api.SyncConfig
 import com.fsck.k9.backend.api.SyncListener
 import com.fsck.k9.mail.AuthenticationFailedException
 import com.fsck.k9.mail.Flag
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.mail.internet.MimeMessage
 import java.util.Date
 import okhttp3.HttpUrl
@@ -192,7 +193,7 @@ class CommandSync(
                     setFlags(messageInfo.flags, true)
                 }
 
-                backendFolder.saveCompleteMessage(message)
+                backendFolder.saveMessage(message, MessageDownloadState.FULL)
             } else {
                 Timber.d("Failed to download message: %s", messageInfo.serverId)
             }

backend/jmap/src/test/java/com/fsck/k9/backend/jmap/InMemoryBackendFolder.kt

@@ -4,6 +4,7 @@ import com.fsck.k9.backend.api.BackendFolder
 import com.fsck.k9.mail.Flag
 import com.fsck.k9.mail.FolderType
 import com.fsck.k9.mail.Message
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.mail.internet.MimeMessage
 import java.util.Date
 import okio.Buffer
@@ -108,16 +109,18 @@ class InMemoryBackendFolder(override var name: String, var type: FolderType) : B
         }
     }
 
-    override fun savePartialMessage(message: Message) {
+    override fun saveMessage(message: Message, downloadState: MessageDownloadState) {
         val messageServerId = checkNotNull(message.uid)
         messages[messageServerId] = message
-        messageFlags[messageServerId] = message.flags.toMutableSet()
+        val flags = message.flags.toMutableSet()
-    }
 
-    override fun saveCompleteMessage(message: Message) {
+        when (downloadState) {
-        val messageServerId = checkNotNull(message.uid)
+            MessageDownloadState.ENVELOPE -> Unit
-        messages[messageServerId] = message
+            MessageDownloadState.PARTIAL -> flags.add(Flag.X_DOWNLOADED_PARTIAL)
-        messageFlags[messageServerId] = message.flags.toMutableSet()
+            MessageDownloadState.FULL -> flags.add(Flag.X_DOWNLOADED_FULL)
+        }
+
+        messageFlags[messageServerId] = flags
     }
 
     override fun getOldestMessageDate(): Date? {

backend/pop3/src/main/java/com/fsck/k9/backend/pop3/CommandDownloadMessage.kt

@@ -3,6 +3,7 @@ package com.fsck.k9.backend.pop3
 import com.fsck.k9.backend.api.BackendStorage
 import com.fsck.k9.mail.FetchProfile.Item.BODY
 import com.fsck.k9.mail.FetchProfile.Item.FLAGS
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.mail.helper.fetchProfileOf
 import com.fsck.k9.mail.store.pop3.Pop3Store
 
@@ -17,7 +18,7 @@ internal class CommandDownloadMessage(private val backendStorage: BackendStorage
             folder.fetch(listOf(message), fetchProfileOf(FLAGS, BODY), null, 0)
 
             val backendFolder = backendStorage.getFolder(folderServerId)
-            backendFolder.saveCompleteMessage(message)
+            backendFolder.saveMessage(message, MessageDownloadState.FULL)
         } finally {
             folder.close()
         }

backend/pop3/src/main/java/com/fsck/k9/backend/pop3/Pop3Sync.java

@@ -10,6 +10,7 @@ import com.fsck.k9.helper.ExceptionHelper;
 import com.fsck.k9.mail.AuthenticationFailedException;
 import com.fsck.k9.mail.FetchProfile;
 import com.fsck.k9.mail.Flag;
+import com.fsck.k9.mail.MessageDownloadState;
 import com.fsck.k9.mail.MessageRetrievalListener;
 import com.fsck.k9.mail.MessagingException;
 import com.fsck.k9.mail.store.pop3.Pop3Folder;
@@ -377,9 +378,9 @@ class Pop3Sync {
                 // Store the updated message locally
                 boolean completeMessage = message.isSet(Flag.X_DOWNLOADED_FULL);
                 if (completeMessage) {
-                    backendFolder.saveCompleteMessage(message);
+                    backendFolder.saveMessage(message, MessageDownloadState.FULL);
                 } else {
-                    backendFolder.savePartialMessage(message);
+                    backendFolder.saveMessage(message, MessageDownloadState.PARTIAL);
                 }
 
                 boolean isOldMessage = isOldMessage(backendFolder, message);
@@ -480,7 +481,7 @@ class Pop3Sync {
                         try {
 
                             // Store the updated message locally
-                            backendFolder.saveCompleteMessage(message);
+                            backendFolder.saveMessage(message, MessageDownloadState.FULL);
                             progress.incrementAndGet();
 
                             // Increment the number of "new messages" if the newly downloaded message is
@@ -603,9 +604,9 @@ class Pop3Sync {
 
         // Store the updated message locally
         if (completeMessage) {
-            backendFolder.saveCompleteMessage(message);
+            backendFolder.saveMessage(message, MessageDownloadState.FULL);
         } else {
-            backendFolder.savePartialMessage(message);
+            backendFolder.saveMessage(message, MessageDownloadState.PARTIAL);
         }
     }
 }

backend/webdav/src/main/java/com/fsck/k9/backend/webdav/CommandDownloadMessage.kt

@@ -3,6 +3,7 @@ package com.fsck.k9.backend.webdav
 import com.fsck.k9.backend.api.BackendStorage
 import com.fsck.k9.mail.FetchProfile.Item.BODY
 import com.fsck.k9.mail.FetchProfile.Item.FLAGS
+import com.fsck.k9.mail.MessageDownloadState
 import com.fsck.k9.mail.helper.fetchProfileOf
 import com.fsck.k9.mail.store.webdav.WebDavStore
 
@@ -16,7 +17,7 @@ internal class CommandDownloadMessage(val backendStorage: BackendStorage, privat
             folder.fetch(listOf(message), fetchProfileOf(FLAGS, BODY), null, 0)
 
             val backendFolder = backendStorage.getFolder(folderServerId)
-            backendFolder.saveCompleteMessage(message)
+            backendFolder.saveMessage(message, MessageDownloadState.FULL)
         } finally {
             folder.close()
         }

backend/webdav/src/main/java/com/fsck/k9/backend/webdav/WebDavSync.java

@@ -10,6 +10,7 @@ import com.fsck.k9.helper.ExceptionHelper;
 import com.fsck.k9.mail.AuthenticationFailedException;
 import com.fsck.k9.mail.FetchProfile;
 import com.fsck.k9.mail.Flag;
+import com.fsck.k9.mail.MessageDownloadState;
 import com.fsck.k9.mail.MessageRetrievalListener;
 import com.fsck.k9.mail.MessagingException;
 import com.fsck.k9.mail.store.webdav.WebDavFolder;
@@ -364,9 +365,9 @@ class WebDavSync {
                 // Store the updated message locally
                 boolean completeMessage = message.isSet(Flag.X_DOWNLOADED_FULL);
                 if (completeMessage) {
-                    backendFolder.saveCompleteMessage(message);
+                    backendFolder.saveMessage(message, MessageDownloadState.FULL);
                 } else {
-                    backendFolder.savePartialMessage(message);
+                    backendFolder.saveMessage(message, MessageDownloadState.PARTIAL);
                 }
 
                 listener.syncNewMessage(folder, messageServerId, false);
@@ -466,7 +467,7 @@ class WebDavSync {
                         try {
 
                             // Store the updated message locally
-                            backendFolder.saveCompleteMessage(message);
+                            backendFolder.saveMessage(message, MessageDownloadState.FULL);
                             progress.incrementAndGet();
 
                             // Increment the number of "new messages" if the newly downloaded message is
@@ -582,9 +583,9 @@ class WebDavSync {
 
         // Store the updated message locally
         if (completeMessage) {
-            backendFolder.saveCompleteMessage(message);
+            backendFolder.saveMessage(message, MessageDownloadState.FULL);
         } else {
-            backendFolder.savePartialMessage(message);
+            backendFolder.saveMessage(message, MessageDownloadState.PARTIAL);
         }
     }
 

mail/common/src/main/java/com/fsck/k9/mail/MessageDownloadState.kt

@@ -0,0 +1,7 @@
+package com.fsck.k9.mail
+
+enum class MessageDownloadState {
+    ENVELOPE,
+    PARTIAL,
+    FULL
+}