Merge pull request #5006 from k9mail/message_id

Change the way moving messages between folders in the database works
This commit is contained in:
cketti 2020-10-16 03:49:54 +02:00 committed by GitHub
commit 40319700d8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 1079 additions and 151 deletions

View file

@ -5,6 +5,7 @@ import com.fsck.k9.CoreResourceProvider
import com.fsck.k9.Preferences
import com.fsck.k9.backend.BackendManager
import com.fsck.k9.mailstore.LocalStoreProvider
import com.fsck.k9.mailstore.MessagesStoreProvider
import com.fsck.k9.notification.NotificationController
import com.fsck.k9.notification.NotificationStrategy
import org.koin.core.qualifier.named
@ -21,6 +22,7 @@ val controllerModule = module {
get<CoreResourceProvider>(),
get<BackendManager>(),
get<Preferences>(),
get<MessagesStoreProvider>(),
get(named("controllerExtensions"))
)
}

View file

@ -68,6 +68,8 @@ import com.fsck.k9.mailstore.LocalFolder;
import com.fsck.k9.mailstore.LocalMessage;
import com.fsck.k9.mailstore.LocalStore;
import com.fsck.k9.mailstore.LocalStoreProvider;
import com.fsck.k9.mailstore.MessageStore;
import com.fsck.k9.mailstore.MessagesStoreProvider;
import com.fsck.k9.mailstore.OutboxState;
import com.fsck.k9.mailstore.OutboxStateRepository;
import com.fsck.k9.mailstore.SendState;
@ -115,6 +117,7 @@ public class MessagingController {
private final LocalStoreProvider localStoreProvider;
private final BackendManager backendManager;
private final Preferences preferences;
private final MessagesStoreProvider messagesStoreProvider;
private final Thread controllerThread;
@ -138,7 +141,8 @@ public class MessagingController {
MessagingController(Context context, NotificationController notificationController,
NotificationStrategy notificationStrategy, LocalStoreProvider localStoreProvider,
UnreadMessageCountProvider unreadMessageCountProvider, CoreResourceProvider resourceProvider,
BackendManager backendManager, Preferences preferences, List<ControllerExtension> controllerExtensions) {
BackendManager backendManager, Preferences preferences, MessagesStoreProvider messagesStoreProvider,
List<ControllerExtension> controllerExtensions) {
this.context = context;
this.notificationController = notificationController;
this.notificationStrategy = notificationStrategy;
@ -147,6 +151,7 @@ public class MessagingController {
this.resourceProvider = resourceProvider;
this.backendManager = backendManager;
this.preferences = preferences;
this.messagesStoreProvider = messagesStoreProvider;
controllerThread = new Thread(new Runnable() {
@Override
@ -1619,12 +1624,14 @@ public class MessagingController {
String sentFolderServerId = sentFolder.getServerId();
Timber.i("Moving sent message to folder '%s' (%d)", sentFolderServerId, sentFolderId);
localFolder.moveMessages(Collections.singletonList(message), sentFolder);
MessageStore messageStore = messagesStoreProvider.getMessageStore(account);
long destinationMessageId = messageStore.moveMessage(message.getDatabaseId(), sentFolderId);
Timber.i("Moved sent message to folder '%s' (%d)", sentFolderServerId, sentFolderId);
if (!sentFolder.isLocalOnly()) {
PendingCommand command = PendingAppend.create(sentFolderId, message.getUid());
String destinationUid = messageStore.getMessageServerId(destinationMessageId);
PendingCommand command = PendingAppend.create(sentFolderId, destinationUid);
queuePendingCommand(account, command);
processPendingCommands(account);
}
@ -1769,11 +1776,13 @@ public class MessagingController {
private void moveOrCopyMessageSynchronous(Account account, long srcFolderId, List<LocalMessage> inMessages,
long destFolderId, MoveOrCopyFlavor operation) {
if (operation == MoveOrCopyFlavor.MOVE_AND_MARK_AS_READ) {
throw new UnsupportedOperationException("MOVE_AND_MARK_AS_READ unsupported");
}
try {
LocalStore localStore = localStoreProvider.getInstance(account);
if ((operation == MoveOrCopyFlavor.MOVE
|| operation == MoveOrCopyFlavor.MOVE_AND_MARK_AS_READ)
&& !isMoveCapable(account)) {
if (operation == MoveOrCopyFlavor.MOVE && !isMoveCapable(account)) {
return;
}
if (operation == MoveOrCopyFlavor.COPY && !isCopyCapable(account)) {
@ -1801,12 +1810,6 @@ public class MessagingController {
List<LocalMessage> messages = localSrcFolder.getMessagesByUids(uids);
if (messages.size() > 0) {
Map<String, Message> origUidMap = new HashMap<>();
for (Message message : messages) {
origUidMap.put(message.getUid(), message);
}
Timber.i("moveOrCopyMessageSynchronous: source folder = %s, %d messages, destination folder = %s, " +
"operation = %s", srcFolderId, messages.size(), destFolderId, operation.name());
@ -1827,25 +1830,35 @@ public class MessagingController {
}
}
} else {
uidMap = localSrcFolder.moveMessages(messages, localDestFolder);
for (Entry<String, Message> entry : origUidMap.entrySet()) {
String origUid = entry.getKey();
Message message = entry.getValue();
for (MessagingListener l : getListeners()) {
l.messageUidChanged(account, srcFolderId, origUid, message.getUid());
}
MessageStore messageStore = messagesStoreProvider.getMessageStore(account);
List<Long> messageIds = new ArrayList<>();
Map<Long, String> messageIdToUidMapping = new HashMap<>();
for (LocalMessage message : messages) {
long messageId = message.getDatabaseId();
messageIds.add(messageId);
messageIdToUidMapping.put(messageId, message.getUid());
}
if (operation == MoveOrCopyFlavor.MOVE_AND_MARK_AS_READ) {
localDestFolder.setFlags(messages, Collections.singleton(Flag.SEEN), true);
unreadCountAffected = true;
Map<Long, Long> moveMessageIdMapping = messageStore.moveMessages(messageIds, destFolderId);
Map<Long, String> destinationMapping =
messageStore.getMessageServerIds(moveMessageIdMapping.values());
uidMap = new HashMap<>();
for (Entry<Long, Long> entry : moveMessageIdMapping.entrySet()) {
long sourceMessageId = entry.getKey();
long destinationMessageId = entry.getValue();
String sourceUid = messageIdToUidMapping.get(sourceMessageId);
String destinationUid = destinationMapping.get(destinationMessageId);
uidMap.put(sourceUid, destinationUid);
}
unsuppressMessages(account, messages);
if (unreadCountAffected) {
// If this move operation changes the unread count, notify the listeners
// that the unread count changed in both the source and destination folder.
int unreadMessageCountSrc = localSrcFolder.getUnreadMessageCount();
int unreadMessageCountDest = localDestFolder.getUnreadMessageCount();
for (MessagingListener l : getListeners()) {
l.folderStatusChanged(account, srcFolderId);
l.folderStatusChanged(account, destFolderId);
@ -2029,10 +2042,33 @@ public class MessagingController {
} else {
Timber.d("Deleting messages in normal folder, moving");
localTrashFolder = localStore.getFolder(trashFolderId);
uidMap = localFolder.moveMessages(messages, localTrashFolder);
MessageStore messageStore = messagesStoreProvider.getMessageStore(account);
List<Long> messageIds = new ArrayList<>();
Map<Long, String> messageIdToUidMapping = new HashMap<>();
for (LocalMessage message : messages) {
long messageId = message.getDatabaseId();
messageIds.add(messageId);
messageIdToUidMapping.put(messageId, message.getUid());
}
Map<Long, Long> moveMessageIdMapping = messageStore.moveMessages(messageIds, trashFolderId);
Map<Long, String> destinationMapping = messageStore.getMessageServerIds(moveMessageIdMapping.values());
uidMap = new HashMap<>();
for (Entry<Long, Long> entry : moveMessageIdMapping.entrySet()) {
long sourceMessageId = entry.getKey();
long destinationMessageId = entry.getValue();
String sourceUid = messageIdToUidMapping.get(sourceMessageId);
String destinationUid = destinationMapping.get(destinationMessageId);
uidMap.put(sourceUid, destinationUid);
}
if (account.isMarkMessageAsReadOnDelete()) {
localTrashFolder.setFlags(messages, Collections.singleton(Flag.SEEN), true);
Collection<Long> destinationMessageIds = moveMessageIdMapping.values();
messageStore.setFlag(destinationMessageIds, Flag.SEEN, true);
}
}
@ -2047,10 +2083,10 @@ public class MessagingController {
Long outboxFolderId = account.getOutboxFolderId();
if (outboxFolderId != null && folderId == outboxFolderId && supportsUpload(account)) {
for (Message message : messages) {
for (String destinationUid : uidMap.values()) {
// If the message was in the Outbox, then it has been copied to local Trash, and has
// to be copied to remote trash
PendingCommand command = PendingAppend.create(trashFolderId, message.getUid());
PendingCommand command = PendingAppend.create(trashFolderId, destinationUid);
queuePendingCommand(account, command);
}
processPendingCommands(account);

View file

@ -8,3 +8,18 @@ fun <T> Cursor.map(block: (Cursor) -> T): List<T> {
block(this)
}
}
fun Cursor.getStringOrNull(columnName: String): String? {
val columnIndex = getColumnIndex(columnName)
return if (isNull(columnIndex)) null else getString(columnIndex)
}
fun Cursor.getIntOrNull(columnName: String): Int? {
val columnIndex = getColumnIndex(columnName)
return if (isNull(columnIndex)) null else getInt(columnIndex)
}
fun Cursor.getLongOrNull(columnName: String): Long? {
val columnIndex = getColumnIndex(columnName)
return if (isNull(columnIndex)) null else getLong(columnIndex)
}

View file

@ -10,4 +10,5 @@ val mailStoreModule = module {
single { SpecialFolderSelectionStrategy() }
single { K9BackendStorageFactory(get(), get(), get(), get()) }
factory { SpecialLocalFoldersCreator(preferences = get(), localStoreProvider = get()) }
single { MessagesStoreProvider(messageStoreFactory = get()) }
}

View file

@ -837,125 +837,6 @@ public class LocalFolder {
return folder.appendMessages(msgs, true);
}
public Map<String, String> moveMessages(final List<LocalMessage> msgs, final LocalFolder destFolder) throws MessagingException {
final Map<String, String> uidMap = new HashMap<>();
try {
this.localStore.getDatabase().execute(false, new DbCallback<Void>() {
@Override
public Void doDbWork(final SQLiteDatabase db) throws WrappedException, UnavailableStorageException {
try {
destFolder.open();
for (LocalMessage message : msgs) {
String oldUID = message.getUid();
Timber.d("Updating folder_id to %s for message with UID %s, " +
"id %d currently in folder %s",
destFolder.getDatabaseId(),
message.getUid(),
message.getDatabaseId(),
getName());
String newUid = K9.LOCAL_UID_PREFIX + UUID.randomUUID().toString();
message.setUid(newUid);
uidMap.put(oldUID, newUid);
// Message threading in the target folder
ThreadInfo threadInfo = destFolder.doMessageThreading(db, message);
/*
* "Move" the message into the new folder
*/
long msgId = message.getDatabaseId();
String[] idArg = new String[] { Long.toString(msgId) };
ContentValues cv = new ContentValues();
cv.put("folder_id", destFolder.getDatabaseId());
cv.put("uid", newUid);
db.update("messages", cv, "id = ?", idArg);
// Create/update entry in 'threads' table for the message in the
// target folder
cv.clear();
cv.put("message_id", msgId);
if (threadInfo.threadId == -1) {
if (threadInfo.rootId != -1) {
cv.put("root", threadInfo.rootId);
}
if (threadInfo.parentId != -1) {
cv.put("parent", threadInfo.parentId);
}
db.insert("threads", null, cv);
} else {
db.update("threads", cv, "id = ?",
new String[] { Long.toString(threadInfo.threadId) });
}
/*
* Add a placeholder message so we won't download the original
* message again if we synchronize before the remote move is
* complete.
*/
// We need to open this folder to get the folder id
open();
cv.clear();
cv.put("uid", oldUID);
cv.putNull("flags");
cv.put("read", 1);
cv.put("deleted", 1);
cv.put("folder_id", databaseId);
cv.put("empty", 0);
String messageId = message.getMessageId();
if (messageId != null) {
cv.put("message_id", messageId);
}
final long newId;
if (threadInfo.msgId != -1) {
// There already existed an empty message in the target folder.
// Let's use it as placeholder.
newId = threadInfo.msgId;
db.update("messages", cv, "id = ?",
new String[] { Long.toString(newId) });
} else {
newId = db.insert("messages", null, cv);
}
/*
* Update old entry in 'threads' table to point to the newly
* created placeholder.
*/
cv.clear();
cv.put("message_id", newId);
db.update("threads", cv, "id = ?",
new String[] { Long.toString(message.getThreadId()) });
}
} catch (MessagingException e) {
throw new WrappedException(e);
}
return null;
}
});
this.localStore.notifyChange();
return uidMap;
} catch (WrappedException e) {
throw(MessagingException) e.getCause();
}
}
/**
* The method differs slightly from the contract; If an incoming message already has a uid
* assigned and it matches the uid of an existing message then this message will replace the
@ -1854,7 +1735,7 @@ public class LocalFolder {
updateFolderColumn("top_group", isInTopGroup ? 1 : 0);
}
private ThreadInfo doMessageThreading(SQLiteDatabase db, Message message) {
public ThreadInfo doMessageThreading(SQLiteDatabase db, Message message) {
long rootId = -1;
long parentId = -1;

View file

@ -0,0 +1,49 @@
package com.fsck.k9.mailstore
import com.fsck.k9.mail.Flag
/**
* Functions for accessing and modifying locally stored messages.
*
* The goal is for this to gradually replace [LocalStore]. Once complete, apps will be able to provide their own
* storage implementation.
*/
interface MessageStore {
/**
* Move a message to another folder.
*
* @return The message's database ID in the destination folder. This will most likely be different from the
* messageId passed to this function.
*/
fun moveMessage(messageId: Long, destinationFolderId: Long): Long
/**
* Move messages to another folder.
*
* @return A mapping of the original message database ID to the new message database ID.
*/
fun moveMessages(messageIds: Collection<Long>, destinationFolderId: Long): Map<Long, Long> {
return messageIds
.map { messageId ->
messageId to moveMessage(messageId, destinationFolderId)
}
.toMap()
}
/**
* Set message flags.
*/
fun setFlag(messageIds: Collection<Long>, flag: Flag, set: Boolean)
/**
* Retrieve the server ID for a given message.
*/
fun getMessageServerId(messageId: Long): String
/**
* Retrieve the server IDs for the given messages.
*
* @return A mapping of the message database ID to the message server ID.
*/
fun getMessageServerIds(messageIds: Collection<Long>): Map<Long, String>
}

View file

@ -0,0 +1,7 @@
package com.fsck.k9.mailstore
import com.fsck.k9.Account
interface MessageStoreFactory {
fun create(account: Account): MessageStore
}

View file

@ -0,0 +1,12 @@
package com.fsck.k9.mailstore
import com.fsck.k9.Account
import java.util.concurrent.ConcurrentHashMap
class MessagesStoreProvider(private val messageStoreFactory: MessageStoreFactory) {
private val messageStores = ConcurrentHashMap<String, MessageStore>()
fun getMessageStore(account: Account): MessageStore {
return messageStores.getOrPut(account.uuid) { messageStoreFactory.create(account) }
}
}

View file

@ -1,6 +1,6 @@
package com.fsck.k9.mailstore;
class ThreadInfo {
public class ThreadInfo {
public final long threadId;
public final long msgId;
public final String messageId;
@ -14,4 +14,4 @@ class ThreadInfo {
this.rootId = rootId;
this.parentId = parentId;
}
}
}

View file

@ -29,6 +29,7 @@ import com.fsck.k9.mailstore.LocalFolder;
import com.fsck.k9.mailstore.LocalMessage;
import com.fsck.k9.mailstore.LocalStore;
import com.fsck.k9.mailstore.LocalStoreProvider;
import com.fsck.k9.mailstore.MessagesStoreProvider;
import com.fsck.k9.mailstore.OutboxState;
import com.fsck.k9.mailstore.OutboxStateRepository;
import com.fsck.k9.mailstore.SendState;
@ -85,6 +86,8 @@ public class MessagingControllerTest extends K9RobolectricTest {
@Mock
private LocalStoreProvider localStoreProvider;
@Mock
private MessagesStoreProvider messagesStoreProvider;
@Mock
private SimpleMessagingListener listener;
@Mock
private LocalSearch search;
@ -140,7 +143,7 @@ public class MessagingControllerTest extends K9RobolectricTest {
controller = new MessagingController(appContext, notificationController, notificationStrategy,
localStoreProvider,
unreadMessageCountProvider, mock(CoreResourceProvider.class), backendManager, preferences,
Collections.<ControllerExtension>emptyList());
messagesStoreProvider, Collections.<ControllerExtension>emptyList());
configureAccount();
configureBackendManager();

View file

@ -16,6 +16,7 @@ dependencies {
testImplementation project(':app:testing')
testImplementation "org.robolectric:robolectric:${versions.robolectric}"
testImplementation "junit:junit:${versions.junit}"
testImplementation "com.google.truth:truth:${versions.truth}"
testImplementation "org.mockito:mockito-core:${versions.mockito}"
testImplementation "com.nhaarman.mockitokotlin2:mockito-kotlin:${versions.mockitoKotlin}"
testImplementation "org.koin:koin-test:${versions.koin}"

View file

@ -1,8 +1,11 @@
package com.fsck.k9.storage
import com.fsck.k9.mailstore.MessageStoreFactory
import com.fsck.k9.mailstore.SchemaDefinitionFactory
import com.fsck.k9.storage.messages.K9MessageStoreFactory
import org.koin.dsl.module
val storageModule = module {
single<SchemaDefinitionFactory> { K9SchemaDefinitionFactory() }
single<MessageStoreFactory> { K9MessageStoreFactory(localStoreProvider = get()) }
}

View file

@ -0,0 +1,21 @@
package com.fsck.k9.storage.messages
internal fun <T> performChunkedOperation(
arguments: Collection<T>,
argumentTransformation: (T) -> String,
chunkSize: Int = 500,
operation: (selectionSet: String, selectionArguments: Array<String>) -> Unit
) {
require(arguments.isNotEmpty()) { "'arguments' must not be empty" }
require(chunkSize in 1..1000) { "'chunkSize' needs to be in 1..1000" }
arguments.asSequence()
.map(argumentTransformation)
.chunked(chunkSize)
.forEach { selectionArguments ->
val selectionSet = selectionArguments.indices
.joinToString(separator = ",", prefix = "IN (", postfix = ")") { "?" }
operation(selectionSet, selectionArguments.toTypedArray())
}
}

View file

@ -0,0 +1,48 @@
package com.fsck.k9.storage.messages
import android.content.ContentValues
import com.fsck.k9.mail.Flag
import com.fsck.k9.mailstore.LockableDatabase
private val SPECIAL_FLAGS = setOf(Flag.SEEN, Flag.FLAGGED, Flag.ANSWERED, Flag.FORWARDED)
internal class FlagMessageOperations(private val lockableDatabase: LockableDatabase) {
fun setFlag(messageIds: Collection<Long>, flag: Flag, set: Boolean) {
require(messageIds.isNotEmpty()) { "'messageIds' must not be empty" }
if (flag in SPECIAL_FLAGS) {
setSpecialFlags(messageIds, flag, set)
} else {
rebuildFlagsColumnValue(messageIds, flag, set)
}
}
private fun setSpecialFlags(messageIds: Collection<Long>, flag: Flag, set: Boolean) {
val columnName = when (flag) {
Flag.SEEN -> "read"
Flag.FLAGGED -> "flagged"
Flag.ANSWERED -> "answered"
Flag.FORWARDED -> "forwarded"
else -> error("Unsupported flag: $flag")
}
val columnValue = if (set) 1 else 0
val contentValues = ContentValues().apply {
put(columnName, columnValue)
}
lockableDatabase.execute(true) { database ->
performChunkedOperation(
arguments = messageIds,
argumentTransformation = Long::toString
) { selectionSet, selectionArguments ->
database.update("messages", contentValues, "id $selectionSet", selectionArguments)
}
}
}
private fun rebuildFlagsColumnValue(messageIds: Collection<Long>, flag: Flag, set: Boolean) {
throw UnsupportedOperationException("not implemented")
}
}

View file

@ -0,0 +1,34 @@
package com.fsck.k9.storage.messages
import com.fsck.k9.mail.Flag
import com.fsck.k9.mailstore.LocalStore
import com.fsck.k9.mailstore.LockableDatabase
import com.fsck.k9.mailstore.MessageStore
// TODO: Remove dependency on LocalStore
class K9MessageStore(private val localStore: LocalStore) : MessageStore {
private val database: LockableDatabase = localStore.database
private val threadMessageOperations = ThreadMessageOperations(localStore)
private val moveMessageOperations = MoveMessageOperations(database, threadMessageOperations)
private val flagMessageOperations = FlagMessageOperations(database)
private val retrieveMessageOperations = RetrieveMessageOperations(database)
override fun moveMessage(messageId: Long, destinationFolderId: Long): Long {
return moveMessageOperations.moveMessage(messageId, destinationFolderId).also {
localStore.notifyChange()
}
}
override fun setFlag(messageIds: Collection<Long>, flag: Flag, set: Boolean) {
flagMessageOperations.setFlag(messageIds, flag, set)
localStore.notifyChange()
}
override fun getMessageServerId(messageId: Long): String {
return retrieveMessageOperations.getMessageServerId(messageId)
}
override fun getMessageServerIds(messageIds: Collection<Long>): Map<Long, String> {
return retrieveMessageOperations.getMessageServerIds(messageIds)
}
}

View file

@ -0,0 +1,13 @@
package com.fsck.k9.storage.messages
import com.fsck.k9.Account
import com.fsck.k9.mailstore.LocalStoreProvider
import com.fsck.k9.mailstore.MessageStore
import com.fsck.k9.mailstore.MessageStoreFactory
class K9MessageStoreFactory(private val localStoreProvider: LocalStoreProvider) : MessageStoreFactory {
override fun create(account: Account): MessageStore {
val localStore = localStoreProvider.getInstance(account)
return K9MessageStore(localStore)
}
}

View file

@ -0,0 +1,121 @@
package com.fsck.k9.storage.messages
import android.content.ContentValues
import android.database.sqlite.SQLiteDatabase
import com.fsck.k9.K9
import com.fsck.k9.helper.getIntOrNull
import com.fsck.k9.helper.getLongOrNull
import com.fsck.k9.helper.getStringOrNull
import com.fsck.k9.mailstore.LockableDatabase
import java.util.UUID
import timber.log.Timber
internal class MoveMessageOperations(
private val database: LockableDatabase,
private val threadMessageOperations: ThreadMessageOperations
) {
fun moveMessage(messageId: Long, destinationFolderId: Long): Long {
Timber.d("Moving message [ID: $messageId] to folder [ID: $destinationFolderId]")
return database.execute(true) { database ->
val threadInfo = threadMessageOperations.createOrUpdateParentThreadEntries(database, messageId, destinationFolderId)
val destinationMessageId = createMessageEntry(database, messageId, destinationFolderId, threadInfo)
threadMessageOperations.createOrUpdateThreadEntry(database, destinationMessageId, threadInfo)
convertOriginalMessageEntryToPlaceholderEntry(database, messageId)
destinationMessageId
}
}
private fun createMessageEntry(
database: SQLiteDatabase,
messageId: Long,
destinationFolderId: Long,
threadInfo: ThreadInfo
): Long {
val destinationUid = K9.LOCAL_UID_PREFIX + UUID.randomUUID().toString()
val contentValues = database.query(
"messages",
arrayOf(
"subject", "date", "flags", "sender_list", "to_list", "cc_list", "bcc_list", "reply_to_list",
"attachment_count", "internal_date", "message_id", "preview_type", "preview", "mime_type",
"normalized_subject_hash", "read", "flagged", "answered", "forwarded", "message_part_id",
"encryption_type"
),
"id = ?",
arrayOf(messageId.toString()),
null, null, null
).use { cursor ->
if (!cursor.moveToFirst()) {
error("Couldn't find local message [ID: $messageId]")
}
ContentValues().apply {
put("uid", destinationUid)
put("folder_id", destinationFolderId)
put("deleted", 0)
put("empty", 0)
put("subject", cursor.getStringOrNull("subject"))
put("date", cursor.getLongOrNull("date"))
put("flags", cursor.getStringOrNull("flags"))
put("sender_list", cursor.getStringOrNull("sender_list"))
put("to_list", cursor.getStringOrNull("to_list"))
put("cc_list", cursor.getStringOrNull("cc_list"))
put("bcc_list", cursor.getStringOrNull("bcc_list"))
put("reply_to_list", cursor.getStringOrNull("reply_to_list"))
put("attachment_count", cursor.getIntOrNull("attachment_count"))
put("internal_date", cursor.getLongOrNull("internal_date"))
put("message_id", cursor.getStringOrNull("message_id"))
put("preview_type", cursor.getStringOrNull("preview_type"))
put("preview", cursor.getStringOrNull("preview"))
put("mime_type", cursor.getStringOrNull("mime_type"))
put("normalized_subject_hash", cursor.getLongOrNull("normalized_subject_hash"))
put("read", cursor.getIntOrNull("read"))
put("flagged", cursor.getIntOrNull("flagged"))
put("answered", cursor.getIntOrNull("answered"))
put("forwarded", cursor.getIntOrNull("forwarded"))
put("message_part_id", cursor.getLongOrNull("message_part_id"))
put("encryption_type", cursor.getStringOrNull("encryption_type"))
}
}
val placeHolderMessageId = threadInfo.messageId
return if (placeHolderMessageId != null) {
database.update("messages", contentValues, "id = ?", arrayOf(placeHolderMessageId.toString()))
placeHolderMessageId
} else {
database.insert("messages", null, contentValues)
}
}
private fun convertOriginalMessageEntryToPlaceholderEntry(database: SQLiteDatabase, messageId: Long) {
val contentValues = ContentValues().apply {
put("deleted", 1)
put("empty", 0)
put("read", 1)
putNull("subject")
putNull("date")
putNull("flags")
putNull("sender_list")
putNull("to_list")
putNull("cc_list")
putNull("bcc_list")
putNull("reply_to_list")
putNull("attachment_count")
putNull("internal_date")
put("preview_type", "none")
putNull("preview")
putNull("mime_type")
putNull("normalized_subject_hash")
putNull("flagged")
putNull("answered")
putNull("forwarded")
putNull("message_part_id")
putNull("encryption_type")
}
database.update("messages", contentValues, "id = ?", arrayOf(messageId.toString()))
}
}

View file

@ -0,0 +1,57 @@
package com.fsck.k9.storage.messages
import com.fsck.k9.mail.MessagingException
import com.fsck.k9.mailstore.LockableDatabase
internal class RetrieveMessageOperations(private val lockableDatabase: LockableDatabase) {
fun getMessageServerId(messageId: Long): String {
return lockableDatabase.execute(false) { database ->
database.query(
"messages",
arrayOf("uid"),
"id = ?",
arrayOf(messageId.toString()),
null, null, null
).use { cursor ->
if (cursor.moveToFirst()) {
cursor.getString(0)
} else {
throw MessagingException("Message [ID: $messageId] not found in database")
}
}
}
}
fun getMessageServerIds(messageIds: Collection<Long>): Map<Long, String> {
if (messageIds.isEmpty()) return emptyMap()
return lockableDatabase.execute(false) { database ->
val databaseIdToServerIdMapping = mutableMapOf<Long, String>()
performChunkedOperation(
arguments = messageIds,
argumentTransformation = Long::toString
) { selectionSet, selectionArguments ->
database.query(
"messages",
arrayOf("id", "uid"),
"id $selectionSet",
selectionArguments,
null,
null,
null
).use { cursor ->
while (cursor.moveToNext()) {
val databaseId = cursor.getLong(0)
val serverId = cursor.getString(1)
databaseIdToServerIdMapping[databaseId] = serverId
}
}
Unit
}
databaseIdToServerIdMapping
}
}
}

View file

@ -0,0 +1,91 @@
package com.fsck.k9.storage.messages
import android.content.ContentValues
import android.database.sqlite.SQLiteDatabase
import com.fsck.k9.mailstore.LocalFolder
import com.fsck.k9.mailstore.LocalMessage
import com.fsck.k9.mailstore.LocalStore
import com.fsck.k9.mailstore.ThreadInfo as LegacyThreadInfo
// TODO: Remove dependency on LocalStore
internal class ThreadMessageOperations(private val localStore: LocalStore) {
fun createOrUpdateParentThreadEntries(
database: SQLiteDatabase,
messageId: Long,
destinationFolderId: Long
): ThreadInfo {
val message = getLocalMessage(database, messageId) ?: error("Couldn't find local message [ID: $messageId]")
val destinationFolder = getLocalFolder(destinationFolderId)
val legacyThreadInfo: LegacyThreadInfo = destinationFolder.doMessageThreading(database, message)
return legacyThreadInfo.toThreadInfo()
}
fun createOrUpdateThreadEntry(
database: SQLiteDatabase,
destinationMessageId: Long,
threadInfo: ThreadInfo
) {
val contentValues = ContentValues()
contentValues.put("message_id", destinationMessageId)
if (threadInfo.threadId == null) {
if (threadInfo.rootId != null) {
contentValues.put("root", threadInfo.rootId)
}
if (threadInfo.parentId != null) {
contentValues.put("parent", threadInfo.parentId)
}
database.insert("threads", null, contentValues)
} else {
database.update("threads", contentValues, "id = ?", arrayOf(threadInfo.threadId.toString()))
}
}
private fun getLocalMessage(database: SQLiteDatabase, messageId: Long): LocalMessage? {
return database.query(
"messages",
arrayOf("uid", "folder_id"),
"id = ?",
arrayOf(messageId.toString()),
null, null, null
).use { cursor ->
if (cursor.moveToFirst()) {
val uid = cursor.getString(0)
val folderId = cursor.getLong(1)
val folder = getLocalFolder(folderId)
folder.getMessage(uid)
} else {
null
}
}
}
private fun getLocalFolder(destinationFolderId: Long): LocalFolder {
val destinationFolder = localStore.getFolder(destinationFolderId)
destinationFolder.open()
return destinationFolder
}
private fun LegacyThreadInfo.toThreadInfo(): ThreadInfo {
return ThreadInfo(
threadId = threadId.minusOneToNull(),
messageId = msgId.minusOneToNull(),
messageIdHeader = messageId,
rootId = rootId.minusOneToNull(),
parentId = parentId.minusOneToNull()
)
}
private fun Long.minusOneToNull() = if (this == -1L) null else this
}
internal data class ThreadInfo(
val threadId: Long?,
val messageId: Long?,
val messageIdHeader: String,
val rootId: Long?,
val parentId: Long?
)

View file

@ -0,0 +1,112 @@
package com.fsck.k9.storage.messages
import com.google.common.truth.Truth.assertThat
import org.junit.Assert.fail
import org.junit.Test
class ChunkedDatabaseOperationsTest {
@Test(expected = IllegalArgumentException::class)
fun `empty list`() {
performChunkedOperation(
arguments = emptyList(),
argumentTransformation = Int::toString,
operation = ::failCallback
)
}
@Test(expected = IllegalArgumentException::class)
fun `chunkSize = 0`() {
performChunkedOperation(
arguments = listOf(1),
argumentTransformation = Int::toString,
chunkSize = 0,
operation = ::failCallback
)
}
@Test(expected = IllegalArgumentException::class)
fun `chunkSize = 1001`() {
performChunkedOperation(
arguments = listOf(1),
argumentTransformation = Int::toString,
chunkSize = 1001,
operation = ::failCallback
)
}
@Test
fun `single item`() {
val chunks = mutableListOf<Pair<String, Array<String>>>()
performChunkedOperation(
arguments = listOf(1),
argumentTransformation = Int::toString
) { selectionSet, selectionArguments ->
chunks.add(selectionSet to selectionArguments)
Unit
}
assertThat(chunks).hasSize(1)
with(chunks.first()) {
assertThat(first).isEqualTo("IN (?)")
assertThat(second).isEqualTo(arrayOf("1"))
}
}
@Test
fun `2 items with chunk size of 1`() {
val chunks = mutableListOf<Pair<String, Array<String>>>()
performChunkedOperation(
arguments = listOf(1, 2),
argumentTransformation = Int::toString,
chunkSize = 1
) { selectionSet, selectionArguments ->
chunks.add(selectionSet to selectionArguments)
Unit
}
assertThat(chunks).hasSize(2)
with(chunks[0]) {
assertThat(first).isEqualTo("IN (?)")
assertThat(second).isEqualTo(arrayOf("1"))
}
with(chunks[1]) {
assertThat(first).isEqualTo("IN (?)")
assertThat(second).isEqualTo(arrayOf("2"))
}
}
@Test
fun `14 items with chunk size of 5`() {
val chunks = mutableListOf<Pair<String, Array<String>>>()
performChunkedOperation(
arguments = (1..14).toList(),
argumentTransformation = Int::toString,
chunkSize = 5
) { selectionSet, selectionArguments ->
chunks.add(selectionSet to selectionArguments)
Unit
}
assertThat(chunks).hasSize(3)
with(chunks[0]) {
assertThat(first).isEqualTo("IN (?,?,?,?,?)")
assertThat(second).isEqualTo(arrayOf("1", "2", "3", "4", "5"))
}
with(chunks[1]) {
assertThat(first).isEqualTo("IN (?,?,?,?,?)")
assertThat(second).isEqualTo(arrayOf("6", "7", "8", "9", "10"))
}
with(chunks[2]) {
assertThat(first).isEqualTo("IN (?,?,?,?)")
assertThat(second).isEqualTo(arrayOf("11", "12", "13", "14"))
}
}
@Suppress("UNUSED_PARAMETER")
private fun failCallback(selectionSet: String, selectionArguments: Array<String>) {
fail("'operation' callback called when it shouldn't")
}
}

View file

@ -0,0 +1,56 @@
package com.fsck.k9.storage.messages
import com.fsck.k9.mail.Flag
import com.fsck.k9.storage.RobolectricTest
import com.google.common.truth.Truth.assertThat
import org.junit.Test
class FlagMessageOperationsTest : RobolectricTest() {
private val sqliteDatabase = createDatabase()
private val lockableDatabase = createLockableDatabaseMock(sqliteDatabase)
private val flagMessageOperations = FlagMessageOperations(lockableDatabase)
@Test(expected = IllegalArgumentException::class)
fun `empty messageIds list`() {
flagMessageOperations.setFlag(emptyList(), Flag.SEEN, true)
}
@Test
fun `mark one message as answered`() {
val messageId = sqliteDatabase.createMessage(folderId = 1, uid = "uid1", answered = false)
sqliteDatabase.createMessage(folderId = 1, uid = "uid2", answered = false)
val messageIds = listOf(messageId)
flagMessageOperations.setFlag(messageIds, Flag.ANSWERED, true)
val messages = sqliteDatabase.readMessages()
val message = messages.find { it.id == messageId } ?: error("Original message not found")
assertThat(message.answered).isEqualTo(1)
val otherMessages = messages.filterNot { it.id == messageId }
assertThat(otherMessages).hasSize(1)
assertThat(otherMessages.all { it.answered == 0 }).isTrue()
}
@Test
fun `mark multiple messages as read`() {
val messageId1 = sqliteDatabase.createMessage(folderId = 1, uid = "uid1", read = false)
val messageId2 = sqliteDatabase.createMessage(folderId = 1, uid = "uid2", read = false)
val messageId3 = sqliteDatabase.createMessage(folderId = 1, uid = "uid3", read = false)
sqliteDatabase.createMessage(folderId = 1, uid = "uidx", read = false)
val messageIds = listOf(messageId1, messageId2, messageId3)
flagMessageOperations.setFlag(messageIds, Flag.SEEN, true)
val messages = sqliteDatabase.readMessages()
val affectedMessages = messages.filter { it.id in messageIds }
assertThat(affectedMessages).hasSize(3)
assertThat(affectedMessages.all { it.read == 1 }).isTrue()
val otherMessages = messages.filterNot { it.id in messageIds }
assertThat(otherMessages).hasSize(1)
assertThat(otherMessages.all { it.read == 0 }).isTrue()
}
}

View file

@ -0,0 +1,160 @@
package com.fsck.k9.storage.messages
import android.content.ContentValues
import android.database.sqlite.SQLiteDatabase
import com.fsck.k9.helper.getIntOrNull
import com.fsck.k9.helper.getLongOrNull
import com.fsck.k9.helper.getStringOrNull
import com.fsck.k9.helper.map
import com.fsck.k9.mailstore.DatabasePreviewType
import com.fsck.k9.mailstore.LockableDatabase
import com.fsck.k9.mailstore.MigrationsHelper
import com.fsck.k9.storage.K9SchemaDefinitionFactory
import com.nhaarman.mockitokotlin2.any
import com.nhaarman.mockitokotlin2.doAnswer
import com.nhaarman.mockitokotlin2.mock
import org.mockito.ArgumentMatchers
fun createLockableDatabaseMock(sqliteDatabase: SQLiteDatabase): LockableDatabase {
return mock {
on { execute(ArgumentMatchers.anyBoolean(), any<LockableDatabase.DbCallback<Any>>()) } doAnswer { stubbing ->
val callback: LockableDatabase.DbCallback<Any> = stubbing.getArgument(1)
callback.doDbWork(sqliteDatabase)
}
}
}
fun createDatabase(): SQLiteDatabase {
val migrationsHelper = mock<MigrationsHelper>()
val sqliteDatabase = SQLiteDatabase.create(null)
val schemaDefinitionFactory = K9SchemaDefinitionFactory()
val schemaDefinition = schemaDefinitionFactory.createSchemaDefinition(migrationsHelper)
schemaDefinition.doDbUpgrade(sqliteDatabase)
return sqliteDatabase
}
fun SQLiteDatabase.createMessage(
deleted: Boolean = false,
folderId: Long,
uid: String,
subject: String = "",
date: Long = 0L,
flags: String = "",
senderList: String = "",
toList: String = "",
ccList: String = "",
bccList: String = "",
replyToList: String = "",
attachmentCount: Int = 0,
internalDate: Long = 0L,
messageId: String = "",
previewType: DatabasePreviewType = DatabasePreviewType.NONE,
preview: String = "",
mimeType: String = "text/plain",
normalizedSubjectHash: Long = 0L,
empty: Boolean = false,
read: Boolean = false,
flagged: Boolean = false,
answered: Boolean = false,
forwarded: Boolean = false,
messagePartId: Long = 0L,
encryptionType: String? = null
): Long {
val values = ContentValues().apply {
put("deleted", if (deleted) 1 else 0)
put("folder_id", folderId)
put("uid", uid)
put("subject", subject)
put("date", date)
put("flags", flags)
put("sender_list", senderList)
put("to_list", toList)
put("cc_list", ccList)
put("bcc_list", bccList)
put("reply_to_list", replyToList)
put("attachment_count", attachmentCount)
put("internal_date", internalDate)
put("message_id", messageId)
put("preview_type", previewType.databaseValue)
put("preview", preview)
put("mime_type", mimeType)
put("normalized_subject_hash", normalizedSubjectHash)
put("empty", if (empty) 1 else 0)
put("read", if (read) 1 else 0)
put("flagged", if (flagged) 1 else 0)
put("answered", if (answered) 1 else 0)
put("forwarded", if (forwarded) 1 else 0)
put("message_part_id", messagePartId)
put("encryption_type", encryptionType)
}
return insert("messages", null, values)
}
fun SQLiteDatabase.readMessages(): List<MessageEntry> {
val cursor = rawQuery("SELECT * FROM messages", null)
return cursor.use {
cursor.map {
MessageEntry(
id = cursor.getLongOrNull("id"),
deleted = cursor.getIntOrNull("deleted"),
folderId = cursor.getLongOrNull("folder_id"),
uid = cursor.getStringOrNull("uid"),
subject = cursor.getStringOrNull("subject"),
date = cursor.getLongOrNull("date"),
flags = cursor.getStringOrNull("flags"),
senderList = cursor.getStringOrNull("sender_list"),
toList = cursor.getStringOrNull("to_list"),
ccList = cursor.getStringOrNull("cc_list"),
bccList = cursor.getStringOrNull("bcc_list"),
replyToList = cursor.getStringOrNull("reply_to_list"),
attachmentCount = cursor.getIntOrNull("attachment_count"),
internalDate = cursor.getLongOrNull("internal_date"),
messageId = cursor.getStringOrNull("message_id"),
previewType = cursor.getStringOrNull("preview_type"),
preview = cursor.getStringOrNull("preview"),
mimeType = cursor.getStringOrNull("mime_type"),
normalizedSubjectHash = cursor.getLongOrNull("normalized_subject_hash"),
empty = cursor.getIntOrNull("empty"),
read = cursor.getIntOrNull("read"),
flagged = cursor.getIntOrNull("flagged"),
answered = cursor.getIntOrNull("answered"),
forwarded = cursor.getIntOrNull("forwarded"),
messagePartId = cursor.getLongOrNull("message_part_id"),
encryptionType = cursor.getStringOrNull("encryption_type")
)
}
}
}
data class MessageEntry(
val id: Long?,
val deleted: Int?,
val folderId: Long?,
val uid: String?,
val subject: String?,
val date: Long?,
val flags: String?,
val senderList: String?,
val toList: String?,
val ccList: String?,
val bccList: String?,
val replyToList: String?,
val attachmentCount: Int?,
val internalDate: Long?,
val messageId: String?,
val previewType: String?,
val preview: String?,
val mimeType: String?,
val normalizedSubjectHash: Long?,
val empty: Int?,
val read: Int?,
val flagged: Int?,
val answered: Int?,
val forwarded: Int?,
val messagePartId: Long?,
val encryptionType: String?
)

View file

@ -0,0 +1,157 @@
package com.fsck.k9.storage.messages
import com.fsck.k9.K9
import com.fsck.k9.storage.RobolectricTest
import com.google.common.truth.Truth.assertThat
import com.nhaarman.mockitokotlin2.any
import com.nhaarman.mockitokotlin2.doReturn
import com.nhaarman.mockitokotlin2.mock
import org.junit.Assert.fail as junitFail
import org.junit.Test
import org.mockito.ArgumentMatchers.anyLong
private const val SOURCE_FOLDER_ID = 3L
private const val DESTINATION_FOLDER_ID = 23L
private const val MESSAGE_ID_HEADER = "<00000000-0000-4000-0000-000000000000@domain.example>"
class MoveMessageOperationsTest : RobolectricTest() {
val sqliteDatabase = createDatabase()
val lockableDatabase = createLockableDatabaseMock(sqliteDatabase)
@Test
fun `move message not part of a thread`() {
val originalMessageId = sqliteDatabase.createMessage(
folderId = SOURCE_FOLDER_ID,
uid = "uid1",
subject = "Move me",
messageId = MESSAGE_ID_HEADER
)
val originalMessage = sqliteDatabase.readMessages().first()
val messageThreader = createMessageThreader(
ThreadInfo(
threadId = null,
messageId = null,
messageIdHeader = MESSAGE_ID_HEADER,
rootId = null,
parentId = null
)
)
val moveMessageOperations = MoveMessageOperations(lockableDatabase, messageThreader)
val destinationMessageId = moveMessageOperations.moveMessage(
messageId = originalMessageId,
destinationFolderId = DESTINATION_FOLDER_ID
)
val messages = sqliteDatabase.readMessages()
assertThat(messages).hasSize(2)
val sourceMessage = messages.find { it.id == originalMessageId }
?: fail("Original message not found")
assertThat(sourceMessage.folderId).isEqualTo(SOURCE_FOLDER_ID)
assertThat(sourceMessage.uid).isEqualTo("uid1")
assertThat(sourceMessage.messageId).isEqualTo(MESSAGE_ID_HEADER)
assertPlaceholderEntry(sourceMessage)
val destinationMessage = messages.find { it.id == destinationMessageId }
?: fail("Destination message not found")
assertThat(destinationMessage.uid).startsWith(K9.LOCAL_UID_PREFIX)
assertThat(destinationMessage).isEqualTo(
originalMessage.copy(
id = destinationMessageId,
folderId = DESTINATION_FOLDER_ID,
uid = destinationMessage.uid,
deleted = 0,
empty = 0
)
)
}
@Test
fun `move message when destination has empty message entry`() {
val originalMessageId = sqliteDatabase.createMessage(
folderId = SOURCE_FOLDER_ID,
uid = "uid1",
subject = "Move me",
messageId = MESSAGE_ID_HEADER,
read = false
)
val originalMessage = sqliteDatabase.readMessages().first()
val placeholderMessageId = sqliteDatabase.createMessage(
empty = true,
folderId = DESTINATION_FOLDER_ID,
messageId = MESSAGE_ID_HEADER,
uid = ""
)
val messageThreader = createMessageThreader(
ThreadInfo(
threadId = null,
messageId = placeholderMessageId,
messageIdHeader = MESSAGE_ID_HEADER,
rootId = null,
parentId = null
)
)
val moveMessageOperations = MoveMessageOperations(lockableDatabase, messageThreader)
val destinationMessageId = moveMessageOperations.moveMessage(
messageId = originalMessageId,
destinationFolderId = DESTINATION_FOLDER_ID
)
val messages = sqliteDatabase.readMessages()
assertThat(messages).hasSize(2)
val sourceMessage = messages.find { it.id == originalMessageId }
?: fail("Original message not found in database")
assertThat(sourceMessage.folderId).isEqualTo(SOURCE_FOLDER_ID)
assertThat(sourceMessage.uid).isEqualTo("uid1")
assertThat(sourceMessage.messageId).isEqualTo(MESSAGE_ID_HEADER)
assertPlaceholderEntry(sourceMessage)
val destinationMessage = messages.find { it.id == destinationMessageId }
?: fail("Destination message not found in database")
assertThat(destinationMessage.uid).startsWith(K9.LOCAL_UID_PREFIX)
assertThat(destinationMessage).isEqualTo(
originalMessage.copy(
id = destinationMessageId,
folderId = DESTINATION_FOLDER_ID,
uid = destinationMessage.uid,
deleted = 0,
empty = 0
)
)
}
private fun createMessageThreader(threadInfo: ThreadInfo): ThreadMessageOperations {
return mock {
on { createOrUpdateParentThreadEntries(any(), anyLong(), anyLong()) } doReturn threadInfo
}
}
private fun assertPlaceholderEntry(message: MessageEntry) {
assertThat(message.deleted).isEqualTo(1)
assertThat(message.empty).isEqualTo(0)
assertThat(message.read).isEqualTo(1)
assertThat(message.date).isNull()
assertThat(message.flags).isNull()
assertThat(message.senderList).isNull()
assertThat(message.toList).isNull()
assertThat(message.ccList).isNull()
assertThat(message.bccList).isNull()
assertThat(message.replyToList).isNull()
assertThat(message.attachmentCount).isNull()
assertThat(message.internalDate).isNull()
assertThat(message.previewType).isEqualTo("none")
assertThat(message.preview).isNull()
assertThat(message.mimeType).isNull()
assertThat(message.normalizedSubjectHash).isNull()
assertThat(message.flagged).isNull()
assertThat(message.answered).isNull()
assertThat(message.forwarded).isNull()
assertThat(message.messagePartId).isNull()
assertThat(message.encryptionType).isNull()
}
private fun fail(message: String): Nothing = junitFail(message) as Nothing
}

View file

@ -0,0 +1,48 @@
package com.fsck.k9.storage.messages
import com.fsck.k9.mail.MessagingException
import com.fsck.k9.storage.RobolectricTest
import com.google.common.truth.Truth.assertThat
import org.junit.Test
class RetrieveMessageOperationsTest : RobolectricTest() {
private val sqliteDatabase = createDatabase()
private val lockableDatabase = createLockableDatabaseMock(sqliteDatabase)
private val retrieveMessageOperations = RetrieveMessageOperations(lockableDatabase)
@Test(expected = MessagingException::class)
fun `get message server id of non-existent message`() {
retrieveMessageOperations.getMessageServerId(42)
}
@Test
fun `get message server id`() {
sqliteDatabase.createMessage(folderId = 1, uid = "uid1")
val messageId = sqliteDatabase.createMessage(folderId = 1, uid = "uid2")
val messageServerId = retrieveMessageOperations.getMessageServerId(messageId)
assertThat(messageServerId).isEqualTo("uid2")
}
@Test
fun `get message server ids`() {
val messageId1 = sqliteDatabase.createMessage(folderId = 1, uid = "uid1")
val messageId2 = sqliteDatabase.createMessage(folderId = 1, uid = "uid2")
val messageId3 = sqliteDatabase.createMessage(folderId = 1, uid = "uid3")
val messageId4 = sqliteDatabase.createMessage(folderId = 1, uid = "uid4")
sqliteDatabase.createMessage(folderId = 1, uid = "uid5")
val messageIds = listOf(messageId1, messageId2, messageId3, messageId4)
val databaseIdToServerIdMapping = retrieveMessageOperations.getMessageServerIds(messageIds)
assertThat(databaseIdToServerIdMapping).isEqualTo(
mapOf(
messageId1 to "uid1",
messageId2 to "uid2",
messageId3 to "uid3",
messageId4 to "uid4"
)
)
}
}