Merge pull request #1847 from k9mail/multipart-attachments

Handle attachments with CHILD_PART_CONTAINS_DATA data location

k9mail-library/src/main/java/com/fsck/k9/mail/Message.java

@@ -145,7 +145,7 @@ public abstract class Message implements Part, Body {
 
     public abstract boolean hasAttachments();
 
-    public abstract int getSize();
+    public abstract long getSize();
 
     public void delete(String trashFolderName) throws MessagingException {}
 

k9mail-library/src/main/java/com/fsck/k9/mail/internet/MimeMessage.java

@@ -182,7 +182,7 @@ public class MimeMessage extends Message {
     }
 
     @Override
-    public int getSize() {
+    public long getSize() {
         return mSize;
     }
 

k9mail-library/src/main/java/com/fsck/k9/mail/store/webdav/WebDavFolder.java

@@ -655,7 +655,11 @@ class WebDavFolder extends Folder<WebDavMessage> {
             try {
                 ByteArrayOutputStream out;
 
-                out = new ByteArrayOutputStream(message.getSize());
+                long size = message.getSize();
+                if (size > Integer.MAX_VALUE) {
+                    throw new MessagingException("message size > Integer.MAX_VALUE!");
+                }
+                out = new ByteArrayOutputStream((int) size);
 
                 open(Folder.OPEN_MODE_RW);
                 EOLConvertingOutputStream msgOut = new EOLConvertingOutputStream(

k9mail/src/main/java/com/fsck/k9/mailstore/LocalFolder.java

@@ -740,7 +740,7 @@ public class LocalFolder extends Folder<LocalMessage> implements Serializable {
                 ((Multipart) parentPart.getBody()).addBodyPart(bodyPart);
                 part = bodyPart;
             } else if (MimeUtility.isMessage(parentMimeType)) {
-                Message innerMessage = new MimeMessage();
+                Message innerMessage = new LocalMimeMessage(getAccountUuid(), message, id);
                 parentPart.setBody(innerMessage);
                 part = innerMessage;
             } else {

k9mail/src/main/java/com/fsck/k9/mailstore/LocalMessage.java

@@ -7,6 +7,7 @@ import java.util.Date;
 import android.content.ContentValues;
 import android.database.Cursor;
 import android.database.sqlite.SQLiteDatabase;
+import android.support.annotation.VisibleForTesting;
 import android.util.Log;
 
 import com.fsck.k9.Account;
@@ -118,7 +119,7 @@ public class LocalMessage extends MimeMessage {
         setFlagInternal(Flag.ANSWERED, answered);
         setFlagInternal(Flag.FORWARDED, forwarded);
 
-        messagePartId = cursor.getLong(22);
+        setMessagePartId(cursor.getLong(22));
         mimeType = cursor.getString(23);
 
         byte[] header = cursor.getBlob(25);
@@ -129,6 +130,11 @@ public class LocalMessage extends MimeMessage {
         }
     }
 
+    @VisibleForTesting
+    public void setMessagePartId(long messagePartId) {
+        this.messagePartId = messagePartId;
+    }
+
     public long getMessagePartId() {
         return messagePartId;
     }

k9mail/src/main/java/com/fsck/k9/mailstore/LocalMimeMessage.java

@@ -0,0 +1,35 @@
+package com.fsck.k9.mailstore;
+
+
+import com.fsck.k9.mail.MessagingException;
+import com.fsck.k9.mail.internet.MimeMessage;
+
+
+public class LocalMimeMessage extends MimeMessage implements LocalPart {
+    private final String accountUuid;
+    private final LocalMessage message;
+    private final long messagePartId;
+
+    public LocalMimeMessage(String accountUuid, LocalMessage message, long messagePartId)
+            throws MessagingException {
+        super();
+        this.accountUuid = accountUuid;
+        this.message = message;
+        this.messagePartId = messagePartId;
+    }
+
+    @Override
+    public String getAccountUuid() {
+        return accountUuid;
+    }
+
+    @Override
+    public long getId() {
+        return messagePartId;
+    }
+
+    @Override
+    public LocalMessage getMessage() {
+        return message;
+    }
+}

k9mail/src/main/java/com/fsck/k9/mailstore/LocalStore.java

@@ -8,6 +8,7 @@ import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -16,6 +17,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Stack;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
@@ -34,11 +36,19 @@ import com.fsck.k9.K9;
 import com.fsck.k9.Preferences;
 import com.fsck.k9.helper.UrlEncodingHelper;
 import com.fsck.k9.helper.Utility;
+import com.fsck.k9.mail.Body;
+import com.fsck.k9.mail.BodyPart;
+import com.fsck.k9.mail.FetchProfile;
+import com.fsck.k9.mail.FetchProfile.Item;
 import com.fsck.k9.mail.Flag;
 import com.fsck.k9.mail.Folder;
 import com.fsck.k9.mail.MessageRetrievalListener;
 import com.fsck.k9.mail.MessagingException;
+import com.fsck.k9.mail.Multipart;
+import com.fsck.k9.mail.Part;
 import com.fsck.k9.mail.Store;
+import com.fsck.k9.mail.internet.MimeMessage;
+import com.fsck.k9.mail.internet.MimeUtility;
 import com.fsck.k9.mailstore.LocalFolder.DataLocation;
 import com.fsck.k9.mailstore.LocalFolder.MoreMessages;
 import com.fsck.k9.mailstore.LockableDatabase.DbCallback;
@@ -54,9 +64,11 @@ import com.fsck.k9.search.LocalSearch;
 import com.fsck.k9.search.SearchSpecification.Attribute;
 import com.fsck.k9.search.SearchSpecification.SearchField;
 import com.fsck.k9.search.SqlQueryBuilder;
+import org.apache.commons.io.IOUtils;
 import org.apache.james.mime4j.codec.Base64InputStream;
 import org.apache.james.mime4j.codec.QuotedPrintableInputStream;
 import org.apache.james.mime4j.util.MimeUtil;
+import org.openintents.openpgp.util.OpenPgpApi.OpenPgpDataSource;
 
 /**
  * <pre>
@@ -64,7 +76,6 @@ import org.apache.james.mime4j.util.MimeUtil;
  * </pre>
  */
 public class LocalStore extends Store implements Serializable {
-
     private static final long serialVersionUID = -5142141896809423072L;
 
     static final String[] EMPTY_STRING_ARRAY = new String[0];
@@ -113,6 +124,14 @@ public class LocalStore extends Store implements Serializable {
 
     static final String[] UID_CHECK_PROJECTION = { "uid" };
 
+    private static final String[] GET_ATTACHMENT_COLS = new String[] { "id", "root", "data_location", "encoding", "data" };
+
+    private static final int ATTACH_PART_ID_INDEX = 0;
+    private static final int ATTACH_ROOT_INDEX = 1;
+    private static final int ATTACH_LOCATION_INDEX = 2;
+    private static final int ATTACH_ENCODING_INDEX = 3;
+    private static final int ATTACH_DATA_INDEX = 4;
+
     /**
      * Maximum number of UIDs to check for existence at once.
      *
@@ -680,59 +699,189 @@ public class LocalStore extends Store implements Serializable {
     }
 
     @Nullable
-    public InputStream getAttachmentInputStream(final String attachmentId) throws MessagingException {
+    public OpenPgpDataSource getAttachmentDataSource(final String partId) throws MessagingException {
-        return database.execute(false, new DbCallback<InputStream>() {
+        return new OpenPgpDataSource() {
             @Override
-            public InputStream doDbWork(final SQLiteDatabase db) throws WrappedException {
+            public void writeTo(OutputStream os) throws IOException {
-                Cursor cursor = db.query("message_parts",
+                writeAttachmentDataToOutputStream(partId, os);
-                        new String[] { "data_location", "data", "encoding" },
-                        "id = ?",
-                        new String[] { attachmentId },
-                        null, null, null);
-                try {
-                    if (!cursor.moveToFirst()) {
-                        return null;
-                    }
-
-                    int location = cursor.getInt(0);
-                    String encoding = cursor.getString(2);
-
-                    InputStream rawInputStream = getRawAttachmentInputStream(cursor, location, attachmentId);
-                    return getDecodingInputStream(rawInputStream, encoding);
-                } finally {
-                    cursor.close();
-                }
             }
-        });
+        };
     }
 
-    @Nullable
+    private void writeAttachmentDataToOutputStream(final String partId, final OutputStream outputStream)
-    private InputStream getRawAttachmentInputStream(Cursor cursor, int location, String attachmentId) {
+            throws IOException {
-        switch (location) {
+        try {
-            case DataLocation.IN_DATABASE: {
+            database.execute(false, new DbCallback<Void>() {
-                byte[] data = cursor.getBlob(1);
+                @Override
-                return new ByteArrayInputStream(data);
+                public Void doDbWork(final SQLiteDatabase db) throws WrappedException, MessagingException {
-            }
+                    Cursor cursor = db.query("message_parts",
-            case DataLocation.ON_DISK: {
+                            GET_ATTACHMENT_COLS,
-                File file = getAttachmentFile(attachmentId);
+                            "id = ?", new String[] { partId },
-                try {
+                            null, null, null);
-                    return new FileInputStream(file);
+                    try {
-                } catch (FileNotFoundException e) {
+                        writeCursorPartsToOutputStream(db, cursor, outputStream);
+                    } catch (IOException e) {
+                        throw new WrappedException(e);
+                    } finally {
+                        Utility.closeQuietly(cursor);
+                    }
+
                     return null;
                 }
-            }
+            });
-            default: {
+        } catch (MessagingException e) {
-                throw new IllegalStateException("No attachment data available");
+            throw new IOException("Got a MessagingException while writing attachment data!", e);
+        } catch (WrappedException e) {
+            throw (IOException) e.getCause();
+        }
+    }
+
+    private void writeCursorPartsToOutputStream(SQLiteDatabase db, Cursor cursor, OutputStream outputStream)
+            throws IOException, MessagingException {
+        while (cursor.moveToNext()) {
+            String partId = cursor.getString(ATTACH_PART_ID_INDEX);
+            int location = cursor.getInt(ATTACH_LOCATION_INDEX);
+
+            if (location == DataLocation.IN_DATABASE || location == DataLocation.ON_DISK) {
+                writeSimplePartToOutputStream(partId, cursor, outputStream);
+            } else if (location == DataLocation.CHILD_PART_CONTAINS_DATA) {
+                writeRawBodyToStream(cursor, db, outputStream);
             }
         }
     }
 
+    private void writeRawBodyToStream(Cursor cursor, SQLiteDatabase db, OutputStream outputStream)
+            throws IOException, MessagingException {
+        long partId = cursor.getLong(ATTACH_PART_ID_INDEX);
+        String rootPart = cursor.getString(ATTACH_ROOT_INDEX);
+        LocalMessage message = loadLocalMessageByRootPartId(db, rootPart);
+
+        if (message == null) {
+            throw new MessagingException("Unable to find message for attachment!");
+        }
+
+        Part part = findPartById(message, partId);
+        if (part == null) {
+            throw new MessagingException("Unable to find attachment part in associated message (db integrity error?)");
+        }
+
+        Body body = part.getBody();
+        if (body == null) {
+            throw new MessagingException("Attachment part isn't available!");
+        }
+
+        body.writeTo(outputStream);
+    }
+
+    static Part findPartById(Part searchRoot, long partId) {
+        if (searchRoot instanceof LocalMessage) {
+            LocalMessage localMessage = (LocalMessage) searchRoot;
+            if (localMessage.getMessagePartId() == partId) {
+                return localMessage;
+            }
+        }
+
+        Stack<Part> partStack = new Stack<>();
+        partStack.add(searchRoot);
+
+        while (!partStack.empty()) {
+            Part part = partStack.pop();
+
+            if (part instanceof LocalPart) {
+                LocalPart localBodyPart = (LocalPart) part;
+                if (localBodyPart.getId() == partId) {
+                    return part;
+                }
+            }
+
+            Body body = part.getBody();
+            if (body instanceof Multipart) {
+                Multipart innerMultipart = (Multipart) body;
+                for (BodyPart innerPart : innerMultipart.getBodyParts()) {
+                    partStack.add(innerPart);
+                }
+            }
+
+            if (body instanceof Part) {
+                partStack.add((Part) body);
+            }
+        }
+
+        return null;
+    }
+
+    private LocalMessage loadLocalMessageByRootPartId(SQLiteDatabase db, String rootPart) throws MessagingException {
+        Cursor cursor = db.query("messages",
+                new String[] { "id" },
+                "message_part_id = ?", new String[] { rootPart },
+                null, null, null);
+        long messageId;
+        try {
+            if (!cursor.moveToFirst()) {
+                return null;
+            }
+
+            messageId = cursor.getLong(0);
+        } finally {
+            Utility.closeQuietly(cursor);
+        }
+
+        return loadLocalMessageByMessageId(messageId);
+    }
+
     @Nullable
-    InputStream getDecodingInputStream(@Nullable final InputStream rawInputStream, @Nullable String encoding) {
+    private LocalMessage loadLocalMessageByMessageId(long messageId) throws MessagingException {
-        if (rawInputStream == null) {
+        Map<String, List<String>> foldersAndUids =
+                getFoldersAndUids(Collections.singletonList(messageId), false);
+        if (foldersAndUids.isEmpty()) {
             return null;
         }
 
+        Map.Entry<String,List<String>> entry = foldersAndUids.entrySet().iterator().next();
+        String folderName = entry.getKey();
+        String uid = entry.getValue().get(0);
+
+        LocalFolder folder = getFolder(folderName);
+        LocalMessage localMessage = folder.getMessage(uid);
+
+        FetchProfile fp = new FetchProfile();
+        fp.add(Item.BODY);
+        folder.fetch(Collections.singletonList(localMessage), fp, null);
+
+        return localMessage;
+    }
+
+    private void writeSimplePartToOutputStream(String partId, Cursor cursor, OutputStream outputStream)
+            throws IOException {
+        int location = cursor.getInt(ATTACH_LOCATION_INDEX);
+        InputStream inputStream = getRawAttachmentInputStream(partId, location, cursor);
+
+        try {
+            String encoding = cursor.getString(ATTACH_ENCODING_INDEX);
+            inputStream = getDecodingInputStream(inputStream, encoding);
+            IOUtils.copy(inputStream, outputStream);
+        } finally {
+            IOUtils.closeQuietly(inputStream);
+        }
+    }
+
+    private InputStream getRawAttachmentInputStream(String partId, int location, Cursor cursor)
+            throws FileNotFoundException {
+        switch (location) {
+            case DataLocation.IN_DATABASE: {
+                byte[] data = cursor.getBlob(ATTACH_DATA_INDEX);
+                return new ByteArrayInputStream(data);
+            }
+            case DataLocation.ON_DISK: {
+                File file = getAttachmentFile(partId);
+                return new FileInputStream(file);
+            }
+            default:
+                throw new IllegalStateException("unhandled case");
+        }
+    }
+
+    InputStream getDecodingInputStream(final InputStream rawInputStream, @Nullable String encoding) {
         if (MimeUtil.ENC_BASE64.equals(encoding)) {
             return new Base64InputStream(rawInputStream) {
                 @Override

k9mail/src/main/java/com/fsck/k9/provider/AttachmentProvider.java

@@ -3,7 +3,6 @@ package com.fsck.k9.provider;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.InputStream;
 import java.util.List;
 
 import android.content.ContentProvider;
@@ -24,6 +23,7 @@ import com.fsck.k9.mail.MessagingException;
 import com.fsck.k9.mail.internet.MimeUtility;
 import com.fsck.k9.mailstore.LocalStore;
 import com.fsck.k9.mailstore.LocalStore.AttachmentInfo;
+import org.openintents.openpgp.util.OpenPgpApi.OpenPgpDataSource;
 import org.openintents.openpgp.util.ParcelFileDescriptorUtil;
 
 
@@ -164,12 +164,12 @@ public class AttachmentProvider extends ContentProvider {
     @Nullable
     private ParcelFileDescriptor openAttachment(String accountUuid, String attachmentId) {
         try {
-            InputStream inputStream = getAttachmentInputStream(accountUuid, attachmentId);
+            OpenPgpDataSource openPgpDataSource = getAttachmentDataSource(accountUuid, attachmentId);
-            if (inputStream == null) {
+            if (openPgpDataSource == null) {
-                Log.e(K9.LOG_TAG, "Error getting InputStream for attachment (part doesn't exist?)");
+                Log.e(K9.LOG_TAG, "Error getting data source for attachment (part doesn't exist?)");
                 return null;
             }
-            return ParcelFileDescriptorUtil.pipeFrom(inputStream);
+            return openPgpDataSource.startPumpThread();
         } catch (MessagingException e) {
             Log.e(K9.LOG_TAG, "Error getting InputStream for attachment", e);
             return null;
@@ -180,9 +180,9 @@ public class AttachmentProvider extends ContentProvider {
     }
 
     @Nullable
-    private InputStream getAttachmentInputStream(String accountUuid, String attachmentId) throws MessagingException {
+    private OpenPgpDataSource getAttachmentDataSource(String accountUuid, String attachmentId) throws MessagingException {
         final Account account = Preferences.getPreferences(getContext()).getAccount(accountUuid);
         LocalStore localStore = LocalStore.getInstance(account, getContext());
-        return localStore.getAttachmentInputStream(attachmentId);
+        return localStore.getAttachmentDataSource(attachmentId);
     }
 }

k9mail/src/test/java/com/fsck/k9/controller/MessagingControllerTest.java

@@ -716,14 +716,14 @@ public class MessagingControllerTest {
     private Message buildSmallNewMessage() {
         Message message = mock(Message.class);
         when(message.olderThan(any(Date.class))).thenReturn(false);
-        when(message.getSize()).thenReturn(MAXIMUM_SMALL_MESSAGE_SIZE);
+        when(message.getSize()).thenReturn((long) MAXIMUM_SMALL_MESSAGE_SIZE);
         return message;
     }
 
     private Message buildLargeNewMessage() {
         Message message = mock(Message.class);
         when(message.olderThan(any(Date.class))).thenReturn(false);
-        when(message.getSize()).thenReturn(MAXIMUM_SMALL_MESSAGE_SIZE + 1);
+        when(message.getSize()).thenReturn((long) (MAXIMUM_SMALL_MESSAGE_SIZE + 1));
         return message;
     }
 

k9mail/src/test/java/com/fsck/k9/mailstore/LocalStoreTest.java

@@ -0,0 +1,83 @@
+package com.fsck.k9.mailstore;
+
+
+import com.fsck.k9.mail.Part;
+import com.fsck.k9.mail.internet.MimeBodyPart;
+import com.fsck.k9.mail.internet.MimeMessage;
+import com.fsck.k9.mail.internet.MimeMultipart;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+
+public class LocalStoreTest {
+
+    @Test
+    public void findPartById__withRootLocalBodyPart() throws Exception {
+        LocalBodyPart searchRoot = new LocalBodyPart(null, null, 123L, -1L);
+
+        Part part = LocalStore.findPartById(searchRoot, 123L);
+
+        assertSame(searchRoot, part);
+    }
+
+    @Test
+    public void findPartById__withRootLocalMessage() throws Exception {
+        LocalMessage searchRoot = new LocalMessage(null, "uid", null);
+        searchRoot.setMessagePartId(123L);
+
+        Part part = LocalStore.findPartById(searchRoot, 123L);
+
+        assertSame(searchRoot, part);
+    }
+
+    @Test
+    public void findPartById__withNestedLocalBodyPart() throws Exception {
+        LocalBodyPart searchRoot = new LocalBodyPart(null, null, 1L, -1L);
+
+        LocalBodyPart needlePart = new LocalBodyPart(null, null, 123L, -1L);
+        MimeMultipart mimeMultipart = new MimeMultipart("boundary");
+        mimeMultipart.addBodyPart(needlePart);
+        searchRoot.setBody(mimeMultipart);
+
+
+        Part part = LocalStore.findPartById(searchRoot, 123L);
+
+
+        assertSame(needlePart, part);
+    }
+
+    @Test
+    public void findPartById__withNestedLocalMessagePart() throws Exception {
+        LocalBodyPart searchRoot = new LocalBodyPart(null, null, 1L, -1L);
+
+        LocalMimeMessage needlePart = new LocalMimeMessage(null, null, 123L);
+        MimeMultipart mimeMultipart = new MimeMultipart("boundary");
+        mimeMultipart.addBodyPart(new MimeBodyPart(needlePart));
+        searchRoot.setBody(mimeMultipart);
+
+
+        Part part = LocalStore.findPartById(searchRoot, 123L);
+
+
+        assertSame(needlePart, part);
+    }
+
+    @Test
+    public void findPartById__withTwoTimesNestedLocalMessagePart() throws Exception {
+        LocalBodyPart searchRoot = new LocalBodyPart(null, null, 1L, -1L);
+
+        LocalMimeMessage needlePart = new LocalMimeMessage(null, null, 123L);
+        MimeMultipart mimeMultipartInner = new MimeMultipart("boundary");
+        mimeMultipartInner.addBodyPart(new MimeBodyPart(needlePart));
+        MimeMultipart mimeMultipart = new MimeMultipart("boundary");
+        mimeMultipart.addBodyPart(new MimeBodyPart(mimeMultipartInner));
+        searchRoot.setBody(mimeMultipart);
+
+
+        Part part = LocalStore.findPartById(searchRoot, 123L);
+
+
+        assertSame(needlePart, part);
+    }
+}

plugins/openpgp-api-lib/openpgp-api/src/main/java/org/openintents/openpgp/util/OpenPgpApi.java

@@ -507,7 +507,7 @@ public class OpenPgpApi {
             return isCancelled;
         }
 
-        private ParcelFileDescriptor startPumpThread() throws IOException {
+        public ParcelFileDescriptor startPumpThread() throws IOException {
             if (writeSidePfd != null) {
                 throw new IllegalStateException("startPumpThread() must only be called once!");
             }